Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

First checkin.

  • Loading branch information...
commit f973106478ac2d396a0242f73e5c8f18dff7b816 1 parent a00f716
@okvivi okvivi authored
Showing with 29,419 additions and 0 deletions.
  1. +2 −0  .gitignore
  2. +15 −0 LICENSE
  3. +13 −0 java/.gitignore
  4. BIN  java/lib/HTTPClient.zip
  5. BIN  java/lib/mysql-connector-java-5.1.7-bin.jar
  6. +38 −0 java/src/ro/vivi/pistruiatul/Candidate.java
  7. +87 −0 java/src/ro/vivi/pistruiatul/CandidatesGroup.java
  8. +311 −0 java/src/ro/vivi/pistruiatul/CdepLaw.java
  9. +144 −0 java/src/ro/vivi/pistruiatul/CdepLaws.java
  10. +206 −0 java/src/ro/vivi/pistruiatul/CdepNominalVote.java
  11. +168 −0 java/src/ro/vivi/pistruiatul/CdepVotesPool.java
  12. +839 −0 java/src/ro/vivi/pistruiatul/DbManager.java
  13. +111 −0 java/src/ro/vivi/pistruiatul/Deputies.java
  14. +595 −0 java/src/ro/vivi/pistruiatul/Deputy.java
  15. +33 −0 java/src/ro/vivi/pistruiatul/ElectionsParties.java
  16. +284 −0 java/src/ro/vivi/pistruiatul/ElectionsSystem2008.java
  17. +99 −0 java/src/ro/vivi/pistruiatul/EuroCandidates2009.java
  18. +105 −0 java/src/ro/vivi/pistruiatul/EuroDeputies.java
  19. +83 −0 java/src/ro/vivi/pistruiatul/EuroParliament.java
  20. +33 −0 java/src/ro/vivi/pistruiatul/GenericLaw.java
  21. +188 −0 java/src/ro/vivi/pistruiatul/InternetsCrawler.java
  22. +64 −0 java/src/ro/vivi/pistruiatul/ListaCatavencu.java
  23. +196 −0 java/src/ro/vivi/pistruiatul/Main.java
  24. +18 −0 java/src/ro/vivi/pistruiatul/PageConsumer.java
  25. +72 −0 java/src/ro/vivi/pistruiatul/Parties.java
  26. +46 −0 java/src/ro/vivi/pistruiatul/Person.java
  27. +71 −0 java/src/ro/vivi/pistruiatul/QvorumEuroRaport2007.java
  28. +44 −0 java/src/ro/vivi/pistruiatul/Seat.java
  29. +81 −0 java/src/ro/vivi/pistruiatul/SeatGroup.java
  30. +139 −0 java/src/ro/vivi/pistruiatul/SenateLaw.java
  31. +91 −0 java/src/ro/vivi/pistruiatul/SenateLaws.java
  32. +44 −0 java/src/ro/vivi/pistruiatul/Senator.java
  33. +102 −0 java/src/ro/vivi/pistruiatul/Senators.java
  34. +95 −0 java/src/ro/vivi/pistruiatul/SingleVote.java
  35. +125 −0 java/src/ro/vivi/pistruiatul/TraficHack.java
  36. +31 −0 java/src/ro/vivi/pistruiatul/Utils.java
  37. +93 −0 java/src/ro/vivi/pistruiatul/Video.java
  38. +376 −0 java/src/ro/vivi/pistruiatul/VotesImporter.java
  39. +194 −0 java/src/ro/vivi/pistruiatul/VotingSession.java
  40. +1,507 −0 mysql/hartapoliticii_pistruiatul.sql
  41. +40 −0 python/.gitignore
  42. 0  python/src/ro/__init__.py
  43. 0  python/src/ro/vivi/__init__.py
  44. +137 −0 python/src/ro/vivi/cdep_crawler/1_get_votes_pages.py
  45. +249 −0 python/src/ro/vivi/cdep_crawler/2_parse_votes.py
  46. +39 −0 python/src/ro/vivi/misc/alegeri_tv_get_presidential_candidates.py
  47. 0  python/src/ro/vivi/news_parser/__init__.py
  48. +96 −0 python/src/ro/vivi/news_parser/crawler.py
  49. +287 −0 python/src/ro/vivi/news_parser/entity_extractor.py
  50. +49 −0 python/src/ro/vivi/news_parser/gov_cabinet.py
  51. +144 −0 python/src/ro/vivi/news_parser/rss_parse_hotnews.py
  52. +130 −0 python/src/ro/vivi/news_parser/rss_parse_mediafax.py
  53. +69 −0 python/src/ro/vivi/news_parser/text_utils.py
  54. +88 −0 python/src/ro/vivi/news_parser/words_processor.py
  55. +131 −0 python/src/ro/vivi/senat_crawler/s01_days_get.py
  56. +132 −0 python/src/ro/vivi/senat_crawler/s02_votes_get.py
  57. +169 −0 python/src/ro/vivi/senat_crawler/s03_votes_parse.py
  58. +32 −0 python/src/ro/vivi/thumb_generator/make_tiny_images.py
  59. +1,488 −0 python/src/ro/vivi/youtube_crawler/atom/__init__.py
  60. BIN  python/src/ro/vivi/youtube_crawler/atom/__init__.pyc
  61. +43 −0 python/src/ro/vivi/youtube_crawler/atom/auth.py
  62. +143 −0 python/src/ro/vivi/youtube_crawler/atom/client.py
  63. +530 −0 python/src/ro/vivi/youtube_crawler/atom/core.py
  64. +339 −0 python/src/ro/vivi/youtube_crawler/atom/data.py
  65. +318 −0 python/src/ro/vivi/youtube_crawler/atom/http.py
  66. BIN  python/src/ro/vivi/youtube_crawler/atom/http.pyc
  67. +535 −0 python/src/ro/vivi/youtube_crawler/atom/http_core.py
  68. BIN  python/src/ro/vivi/youtube_crawler/atom/http_core.pyc
  69. +158 −0 python/src/ro/vivi/youtube_crawler/atom/http_interface.py
  70. BIN  python/src/ro/vivi/youtube_crawler/atom/http_interface.pyc
  71. +132 −0 python/src/ro/vivi/youtube_crawler/atom/mock_http.py
  72. +297 −0 python/src/ro/vivi/youtube_crawler/atom/mock_http_core.py
  73. +243 −0 python/src/ro/vivi/youtube_crawler/atom/mock_service.py
  74. +740 −0 python/src/ro/vivi/youtube_crawler/atom/service.py
  75. BIN  python/src/ro/vivi/youtube_crawler/atom/service.pyc
  76. +117 −0 python/src/ro/vivi/youtube_crawler/atom/token_store.py
  77. BIN  python/src/ro/vivi/youtube_crawler/atom/token_store.pyc
  78. +139 −0 python/src/ro/vivi/youtube_crawler/atom/url.py
  79. BIN  python/src/ro/vivi/youtube_crawler/atom/url.pyc
  80. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/AES.pyd
  81. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/ARC2.pyd
  82. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/ARC4.pyd
  83. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/Blowfish.pyd
  84. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/CAST.pyd
  85. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/DES.pyd
  86. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/DES3.pyd
  87. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/IDEA.pyd
  88. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/RC5.pyd
  89. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/XOR.pyd
  90. +33 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Cipher/__init__.py
  91. +108 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/HMAC.py
  92. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/MD2.pyd
  93. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/MD4.pyd
  94. +13 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/MD5.py
  95. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/RIPEMD.pyd
  96. +11 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/SHA.py
  97. BIN  python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/SHA256.pyd
  98. +24 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Hash/__init__.py
  99. +295 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Protocol/AllOrNothing.py
  100. +229 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Protocol/Chaffing.py
  101. +17 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Protocol/__init__.py
  102. +238 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/DSA.py
  103. +132 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/ElGamal.py
  104. +256 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/RSA.py
  105. +17 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/__init__.py
  106. +172 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/pubkey.py
  107. +170 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/PublicKey/qNEW.py
  108. +342 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Util/RFC1751.py
  109. +16 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Util/__init__.py
  110. +201 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Util/number.py
  111. +421 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Util/randpool.py
  112. +453 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/Util/test.py
  113. +25 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/__init__.py
  114. +38 −0 python/src/ro/vivi/youtube_crawler/gdata/Crypto/test.py
  115. +835 −0 python/src/ro/vivi/youtube_crawler/gdata/__init__.py
  116. BIN  python/src/ro/vivi/youtube_crawler/gdata/__init__.pyc
  117. +20 −0 python/src/ro/vivi/youtube_crawler/gdata/alt/__init__.py
  118. +101 −0 python/src/ro/vivi/youtube_crawler/gdata/alt/app_engine.py
  119. +311 −0 python/src/ro/vivi/youtube_crawler/gdata/alt/appengine.py
  120. +223 −0 python/src/ro/vivi/youtube_crawler/gdata/analytics/__init__.py
  121. +331 −0 python/src/ro/vivi/youtube_crawler/gdata/analytics/service.py
  122. +526 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/__init__.py
  123. +16 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/adminsettings/__init__.py
  124. +476 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/adminsettings/service.py
  125. +15 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/emailsettings/__init__.py
  126. +269 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/emailsettings/service.py
  127. 0  python/src/ro/vivi/youtube_crawler/gdata/apps/groups/__init__.py
  128. +309 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/groups/service.py
  129. +212 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/migration/__init__.py
  130. +129 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/migration/service.py
  131. +553 −0 python/src/ro/vivi/youtube_crawler/gdata/apps/service.py
  132. +952 −0 python/src/ro/vivi/youtube_crawler/gdata/auth.py
  133. BIN  python/src/ro/vivi/youtube_crawler/gdata/auth.pyc
  134. +687 −0 python/src/ro/vivi/youtube_crawler/gdata/base/__init__.py
  135. +256 −0 python/src/ro/vivi/youtube_crawler/gdata/base/service.py
  136. +202 −0 python/src/ro/vivi/youtube_crawler/gdata/blogger/__init__.py
  137. +153 −0 python/src/ro/vivi/youtube_crawler/gdata/blogger/client.py
  138. +146 −0 python/src/ro/vivi/youtube_crawler/gdata/blogger/data.py
  139. +142 −0 python/src/ro/vivi/youtube_crawler/gdata/blogger/service.py
  140. +473 −0 python/src/ro/vivi/youtube_crawler/gdata/books/__init__.py
  141. +266 −0 python/src/ro/vivi/youtube_crawler/gdata/books/service.py
  142. +929 −0 python/src/ro/vivi/youtube_crawler/gdata/calendar/__init__.py
  143. +595 −0 python/src/ro/vivi/youtube_crawler/gdata/calendar/service.py
  144. +747 −0 python/src/ro/vivi/youtube_crawler/gdata/client.py
  145. +136 −0 python/src/ro/vivi/youtube_crawler/gdata/codesearch/__init__.py
  146. +109 −0 python/src/ro/vivi/youtube_crawler/gdata/codesearch/service.py
  147. +741 −0 python/src/ro/vivi/youtube_crawler/gdata/contacts/__init__.py
  148. +432 −0 python/src/ro/vivi/youtube_crawler/gdata/contacts/service.py
  149. +1,159 −0 python/src/ro/vivi/youtube_crawler/gdata/data.py
Sorry, we could not display the entire diff because too many files (1,501) changed.
View
2  .gitignore
@@ -0,0 +1,2 @@
+# Ignore the IntelliJ project files.
+.idea
View
15 LICENSE
@@ -0,0 +1,15 @@
+Harta Politicii Din România
+Copyright (C) 2011 Octavian Costache
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
View
13 java/.gitignore
@@ -0,0 +1,13 @@
+# Dot files that we don't care about.
+*.DS_Store
+.idea/workspace.xml
+.classpath
+Pistruiatul.iml
+.idea/*
+
+# The cached pages from the senat website.
+data.cached
+
+# Don't care about binary files.
+bin
+out
View
BIN  java/lib/HTTPClient.zip
Binary file not shown
View
BIN  java/lib/mysql-connector-java-5.1.7-bin.jar
Binary file not shown
View
38 java/src/ro/vivi/pistruiatul/Candidate.java
@@ -0,0 +1,38 @@
+package ro.vivi.pistruiatul;
+
+/**
+ * A data structure for holding a candidate in the parliamentary elections.
+ * This holds the name, votes and other info about the candidate.
+ * @author vivi
+ *
+ */
+public class Candidate implements Comparable {
+ public int id;
+ public int idperson;
+
+ /** Name of the candidate. */
+ public String name;
+
+ /** Number of votes. */
+ public int votes;
+
+ /** The seat that he was running for. */
+ public Seat runsForSeat;
+
+ /** The seat that he won. */
+ public Seat wonSeat;
+
+ /** The party he belongs to. */
+ public int party;
+
+ public String partyName;
+
+ public int compareTo(Object o) {
+ Candidate c = (Candidate)o;
+ return c.votes - this.votes;
+ }
+
+ public String toString() {
+ return name + "(" + votes + (wonSeat != null ? "/w" : "") + ")";
+ }
+}
View
87 java/src/ro/vivi/pistruiatul/CandidatesGroup.java
@@ -0,0 +1,87 @@
+package ro.vivi.pistruiatul;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+
+/**
+ * Represents a group of candidates.
+ * @author vivi
+ *
+ */
+public class CandidatesGroup {
+ ArrayList<Candidate> candidates = new ArrayList<Candidate>();
+
+ public SeatGroup seatGroup;
+
+ public CandidatesGroup(SeatGroup sg) {
+ this.seatGroup = sg;
+ DbManager.getCandidatesForSeatGroup(sg, candidates);
+ }
+
+ /**
+ * Returns the total number of votes for a particular seat.
+ * @param seat
+ * @return
+ */
+ public int getTotalVotes(Seat seat) {
+ int sum = 0;
+ for (Candidate c : candidates) {
+ sum += c.runsForSeat == seat ? c.votes : 0;
+ }
+ return sum;
+ }
+
+ /**
+ * Returns the candidate with the most votes for a particular seat.
+ */
+ public Candidate getTopDog(Seat seat) {
+ ArrayList<Candidate> list = getSortedCandidates(seat);
+ return list.get(0);
+ }
+
+ /**
+ * Returns a candidate object, by name.
+ * @param name
+ * @return
+ */
+ public Candidate getCandidate(String name) {
+ for (Candidate candidate : candidates) {
+ if (candidate.name.equals(name)) {
+ return candidate;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the most recent winner from the party given as a parameter.
+ * @param seat
+ * @return
+ */
+ public Candidate getMostRecentNonMajorityWinner(String partyName) {
+ ArrayList<Candidate> list = getSortedCandidates(null);
+ Candidate mostRecentWinner = null;
+ for (Candidate candidate : list) {
+ if (candidate.wonSeat != null &&
+ candidate.partyName.equals(partyName) &&
+ candidate.votes < candidate.wonSeat.totalVotes / 2) {
+ mostRecentWinner = candidate;
+ }
+ }
+ return mostRecentWinner;
+ }
+
+ @SuppressWarnings("unchecked")
+ public ArrayList<Candidate> getSortedCandidates(Seat seat) {
+ ArrayList<Candidate> list = new ArrayList<Candidate>();
+
+ for (Candidate c : candidates) {
+ if (seat == null || c.runsForSeat == seat) {
+ list.add(c);
+ }
+ }
+ Collections.sort(list);
+ return list;
+ }
+}
View
311 java/src/ro/vivi/pistruiatul/CdepLaw.java
@@ -0,0 +1,311 @@
+package ro.vivi.pistruiatul;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data model for holding everything that is related to a certain law.
+ * This will be used to load data from the DB if needed, but more likely to dump
+ * data in the database.
+ *
+ * @author vivi
+ */
+public class CdepLaw implements PageConsumer {
+ static Logger log = Logger.getLogger("ro.vivi.pistruiatul.CdepLaw");
+
+ public static String VOTE_URL = "/pls/steno/eVot.Nominal?uniqueId={IDV}";
+ public static String LAW_URL = "/pls/proiecte/upl_pck.proiect?idp={IDP}";
+
+ /** Extracts the party of a deputy from his vote line */
+ Pattern partyPattern = Pattern.compile("<td align=\"center\">(.*)</td>");
+
+ /** Extracts the deputy ID number and his name from the line with his name */
+ Pattern linkDeputyPattern =
+ Pattern.compile("<td><A HREF=\"/pls/parlam/structura\\.mp\\?idm=(\\d*)&" +
+ "cam=(\\d)&leg=(\\d*)\">(.*)</A></td>");
+
+ /** Id of the law */
+ public int id;
+ public String idp;
+ public String link;
+
+ /** This is the id of the vote for this law */
+ public ArrayList<String> idvs;
+ public ArrayList<Long> times;
+
+ /** The type of the law can be Respingere, Aprobare */
+ public ArrayList<String> types;
+
+ /** The number of the law, in the form of NNN/YYYY */
+ public String projectNumber;
+
+ /** The time when this law was voted. */
+ public String dateString;
+
+
+ /** The description of the law, text */
+ public String description;
+
+ private static final int VOTES = 1;
+ private static final int DETAILS = 2;
+ private int nowCrawling = 0;
+
+ private static final int APPROVED = 1;
+ private static final int REJECTED = 2;
+ private static final int ONGOING = 3;
+
+ /**
+ * A link to the container of cdepLaws that has this law. Kindof like if this
+ * were a religious law, I want to know if it's from the Bible or The Coran.
+ */
+ private CdepLaws laws;
+
+ private HashMap<Deputy, String> votes = new HashMap<Deputy, String>();
+
+ /**
+ * Constructor.
+ * @param projectNumber The id of the vote, used to get to the link for that
+ * particular vote and fetch the list of deputies that voted for it.
+ */
+ public CdepLaw(String projectNumber, CdepLaws laws) {
+ this.projectNumber = projectNumber;
+ this.laws = laws;
+
+ idvs = new ArrayList<String>();
+ times = new ArrayList<Long>();
+ types = new ArrayList<String>();
+ }
+
+ /**
+ * Add a vote on this law project. We expect to be one vote per law, but only
+ * crawling the data will tell us.
+ * @param idv
+ */
+ public void addVote(String idv, String type, String datetime) {
+ idvs.add(idv);
+ times.add(new Long(getTimeFromVoteString(datetime)));
+ types.add(type);
+ }
+
+ /**
+ * Given a vote id, go to the site and load the votes on that particular
+ * instance of a vote. If there are more votes on one given law, this should
+ * be refactored to correctly account for all the votes. For now, we make the
+ * assumption that there's one (relevant) vote per law.
+ * @param idv
+ */
+ public void loadVotesFromSite(String idv) {
+ nowCrawling = VOTES;
+ if (!idv.equals(idvs.get(idvs.size() - 1))) {
+ log.warning("CdepLaw " + projectNumber + " error, vote " + idv +
+ " but registered was " + idvs.get(idvs.size() - 1));
+ return;
+ }
+
+ String path = VOTE_URL.replace("{IDV}", idv);
+ InternetsCrawler.enqueue(Main.HOST, path, this);
+ }
+
+ /**
+ * Loads the page with details about this law so that we can parse more
+ * details about a certain law.
+ */
+ public void loadDetailsFromSite() {
+ nowCrawling = DETAILS;
+ String path = LAW_URL.replace("{IDP}", idp);
+ InternetsCrawler.enqueue(Main.HOST, path, this);
+ }
+
+ /**
+ * Stub to redirect the parsing depending on the page that we are currently
+ * fetching.
+ */
+ public void consume(String data) {
+ switch (nowCrawling) {
+ case VOTES:
+ consumeVotesPage(data);
+ break;
+ case DETAILS:
+ consumeDetailsPage(data);
+ break;
+ }
+ }
+
+ private Pattern initiators =
+ Pattern.compile("<tr valign=top><td bgcolor=\"#fff0d8\">" +
+ "Ini.iator:</td>(.*)");
+ private Pattern deputy =
+ Pattern.compile("(.*)/pls/parlam/structura\\.mp\\?idm=(\\d*)&" +
+ "leg=(\\d*)&cam=(\\d)\">(.*)");
+
+ private static final String GOV_INITIATED = "iator:</td><td>Guvern</td></tr>";
+
+ private static final String STATE =
+ "<tr valign=top><td bgcolor=\"#fff0d8\">Stadiu:</td><td>";
+ Pattern isLawPattern =
+ Pattern.compile("<A HREF=\"/pls/legis/legis_pck\\.htp_act\\?" +
+ "nr=(\\d*)&an=(\\d*)\">Lege (\\d*)/(\\d*)</A>$");
+
+ /**
+ * Gets information from the details page.
+ */
+ private void consumeDetailsPage(String data) {
+ String[] lines = data.split("\n");
+ Deputies deputies = laws.getDeputies();
+
+ int lawStatus = 0;
+ int gov = 0;
+ int i = 0;
+ while (i < lines.length) {
+ Matcher initMatcher = initiators.matcher(lines[i]);
+
+ if (lines[i].endsWith(GOV_INITIATED)) {
+ //log.info("CdepLaw " + projectNumber + ": Guvern ");
+ gov++;
+
+ } else if (initMatcher.matches()) {
+ //<A HREF="/pls/parlam/structura.mp?idm=191&leg=2004&cam=2">
+ //Munteanu&nbsp;Ioan</A>
+
+ String[] dudeLines = lines[i].split("</A>");
+ ArrayList<Proponent> props = new ArrayList<Proponent>();
+ StringBuilder proponents = new StringBuilder();
+
+ for (String dudeLine : dudeLines) {
+ //log.info(dudeLine);
+ Matcher d = deputy.matcher(dudeLine);
+ Proponent p;
+ if (d.matches()) {
+ p = new Proponent();
+ p.idm = Integer.parseInt(d.group(2));
+ p.leg = Integer.parseInt(d.group(3));
+ p.chamber = Integer.parseInt(d.group(4));
+
+ props.add(p);
+ proponents.append(p.idm + "/" + p.leg + "/" + p.chamber + " ");
+ } else {
+ if (!dudeLine.equals("</td></tr></table></td></tr>")) {
+ log.warning(dudeLine);
+ }
+ }
+ }
+
+ for (Proponent p : props) {
+ if (p.chamber == 2 && p.leg == 2008) {
+ Deputy dep = deputies.getDeputyForIdm(p.idm);
+ if (dep == null) {
+ log.warning("Could not find " + p.idm + " idp=" + idp);
+ } else {
+ DbManager.insertLawProponent(id, dep.idm, p.chamber, props.size());
+ }
+ }
+ }
+
+ //log.info("CdepLaw " + projectNumber + " " + props.size() + " proponents: " +
+ // proponents);
+ } else if (lines[i].equals(STATE)) {
+ String reason = lines[i + 1];
+
+ Matcher isLawMatcher = isLawPattern.matcher(reason);
+ if (isLawMatcher.matches() ||
+ reason.startsWith("lege trimis\u0103 la promulgare")) {
+ lawStatus = APPROVED;
+ } else if (reason.startsWith("procedur\u0103 legislativ\u0103 �ncetat\u0103") ||
+ reason.startsWith("Sesizare de neconstitu")) {
+ lawStatus = REJECTED;
+ } else {
+ lawStatus = ONGOING; // ongoing
+ //log.info("CdepLaw " + projectNumber + " rejected like this: " +
+ // lines[i+1]);
+ }
+ } else if (lines[i].startsWith("<td class=\"headline\" width=\"100%\">")) {
+ int pos = lines[i].indexOf("<br>") + 4;
+ description = lines[i].substring(pos, lines[i].length() - 4);
+
+ DbManager.updateCdepLawDescription(id, description);
+ }
+ i++;
+ }
+ DbManager.insertLawStatus(id, lawStatus);
+ }
+
+ // Simple inner class to use as a struct;
+ class Proponent {
+ int idm;
+ int chamber;
+ int leg;
+ }
+
+ /**
+ * Parses the page with all the votes on this law/vote.
+ * We will run in trouble here when we will want to parse more than just the
+ * votes for a particular law. :-) We'll worry about that a little later.
+ */
+ private void consumeVotesPage(String data) {
+ String[] lines = data.split("\n");
+ Deputies deputies = laws.getDeputies();
+
+ int i = 0;
+ while (i < lines.length) {
+ Matcher m = linkDeputyPattern.matcher(lines[i]);
+ if (m.matches()) {
+ // We have a deputy link, let's get him out of here.
+ String idm = m.group(1);
+ String name = m.group(4);
+
+ Matcher pm = partyPattern.matcher(lines[++i]);
+ String partyName = pm.matches() ? pm.group(1) : "Not matches";
+
+ long time = times.get(times.size() - 1).longValue();
+ Deputy dep = deputies.contains(name) ?
+ deputies.get(name) : new Deputy(name, idm);
+ dep.setParty("foo", partyName, time);
+
+ if (!deputies.contains(name)) {
+ deputies.add(dep);
+ }
+ // Now that we have the deputy (from the list or not), let's get his
+ // vote
+ String vote = lines[i + 2];
+ votes.put(dep, vote);
+
+ String idv = idvs.get(idvs.size() - 1);
+ DbManager.insertPersonVote(Main.YEAR, this.link, vote, this.id, time,
+ dep);
+ }
+ i++;
+ }
+ log.info("CdepLaw " + this.projectNumber + " has " + votes.size());
+ }
+
+ /**
+ * Sets the date and time of this date, from a string.
+ */
+ private long getTimeFromVoteString(String s) {
+ try {
+ SimpleDateFormat df = new SimpleDateFormat("dd.MM.yyyy hh:mm");
+ return df.parse(s).getTime();
+ } catch (ParseException pe) {
+ pe.printStackTrace();
+ }
+ return 0;
+ }
+
+ /** Sets the description for this law. Usually, by the way it starts, we can
+ * figure out extra information... i think.
+ */
+ public void setDescription(String desc) {
+ description = desc.replace("'", "");
+ }
+
+ @Override
+ public String toString() {
+ return types.get(types.size() - 1) + " " + projectNumber + " " +
+ description;
+ }
+}
View
144 java/src/ro/vivi/pistruiatul/CdepLaws.java
@@ -0,0 +1,144 @@
+package ro.vivi.pistruiatul;
+
+import java.util.HashMap;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data model for the cdepLaws that were voted electronically.
+ * @author vivi
+ */
+public class CdepLaws implements PageConsumer {
+ Logger log = Logger.getLogger("ro.vivi.pistruiatul.CdepLaws");
+
+ /** The url from which we will grab the daily votes. */
+ public static String SEED_PATH =
+ "/pls/steno/eVot.Data?dat={DATE}&cam=2&idl=1";
+
+ /** Keeping track of the cdepLaws, indexed by the id of the law. */
+ HashMap<String, CdepLaw> laws = new HashMap<String, CdepLaw>();
+
+ /**
+ * A reference to Main that will help me get references to other pieces of
+ * this puzzle.
+ */
+ private Main main;
+
+ /**
+ * Initializes some of the much needed data structures.
+ * @param year The year that needs to be loaded.
+ */
+ public CdepLaws(Main main, String year) {
+ this.main = main;
+ loadFromDb(year);
+ }
+
+ /** Loads the deputies from the database */
+ private void loadFromDb(String year) {
+ DbManager.loadCdepLaws(this, year);
+ log.info("CdepLaws loaded from db: " + laws.size());
+ }
+
+ /**
+ * Fetches the list of cdepLaws from the site.
+ */
+ public void loadLawsFromSite() {
+ for (int page = 1; page <= 36; page++) {
+ String path = SEED_PATH.replace("{PAGE}", "" + page);
+ log.info("Fetching page: " + path);
+ InternetsCrawler.enqueue(Main.HOST, path, this);
+ }
+ }
+
+ /**
+ * Enhances the information on cdepLaws by figuring out who proposed a certain law
+ * and if it was rejected or not.
+ */
+ public void enhanceInfo() {
+ for (CdepLaw cdepLaw : laws.values()) {
+ cdepLaw.loadDetailsFromSite();
+ }
+ }
+
+ public CdepLaw getLaw(String projectNumber, String idp) {
+ if (!laws.containsKey(projectNumber)) {
+ CdepLaw cdepLaw = new CdepLaw(projectNumber, this);
+ cdepLaw.idp = idp;
+
+ cdepLaw.link = "http://www.cdep.ro/pls/proiecte/upl_pck.proiect?idp=" + idp;
+ // If the cdepLaw is not already in the hash, put it there
+ laws.put(projectNumber, cdepLaw);
+ cdepLaw.id = DbManager.insertCdepLaw(cdepLaw);
+
+ log.info("CdepLaw " + projectNumber + " id=" + cdepLaw.id);
+
+ cdepLaw.loadDetailsFromSite();
+
+ return cdepLaw;
+ } else {
+ return laws.get(projectNumber);
+ }
+ }
+
+ /**
+ * @inheritDoc
+ */
+ public void consume(String page) {
+ Pattern dateAndTime =
+ Pattern.compile("<a href=\"eVot\\.Nominal\\?idv=([0-9]+)\">(.*)</a>");
+ Pattern numarLink =
+ Pattern.compile("<A HREF=\"/pls/proiecte/upl_pck\\.proiect\\?idp=" +
+ "([0-9]*)\" TARGET=\"PROIECTE\">PL ([0-9/]*)</A>");
+
+ // consume the page that came back.
+ String[] lines = page.split("\n");
+ int i = 0;
+ while (i < lines.length) {
+ String line = lines[i];
+ Matcher m = dateAndTime.matcher(line);
+ if (m.matches()) {
+ // We have a cdepLaw, let's create an object
+ String idv = m.group(1);
+ String datetime = m.group(2);
+
+ // Now I might have a line with the type which can be Adoptare,
+ // Respingere If it doesn't start with <A HREF
+ i += 4;
+ Boolean isFinalVote = lines[i].startsWith("<A HREF");
+ String type = !lines[i].startsWith("<A HREF") ? lines[i++] : "Ordinar";
+
+ // it means it's a cdepLaw
+ Matcher nl = numarLink.matcher(lines[i]);
+ String idp = nl.matches() ? nl.group(1) : "-1";
+ String projectNumber = nl.matches() ? nl.group(2) : "NonFinalVote" + idv;
+
+ if (nl.matches()) {
+ i++;
+ }
+
+ //CdepLaw cdepLaw = cdepLaws.containsKey(projectNumber) ?
+ // cdepLaws.get(projectNumber) : new CdepLaw(projectNumber, this);
+ CdepLaw cdepLaw = getLaw(projectNumber, idp);
+
+ cdepLaw.setDescription(lines[i]);
+ cdepLaw.addVote(idv, type, datetime);
+
+ if (!laws.containsKey(projectNumber)) {
+ // If the cdepLaw is not already in the hash, put it there
+ laws.put(projectNumber, cdepLaw);
+ cdepLaw.id = DbManager.insertCdepLaw(cdepLaw);
+ }
+ cdepLaw.loadVotesFromSite(idv);
+ }
+ i++;
+ }
+ }
+
+ /**
+ * Returns a reference to the pool of deputies.
+ */
+ public Deputies getDeputies() {
+ return main.deputies;
+ }
+}
View
206 java/src/ro/vivi/pistruiatul/CdepNominalVote.java
@@ -0,0 +1,206 @@
+package ro.vivi.pistruiatul;
+
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data model for holding everything that is related to a certain law.
+ * This will be used to load data from the DB if needed, but more likely to dump
+ * data in the database.
+ *
+ * @author vivi
+ */
+public class CdepNominalVote extends VotingSession implements PageConsumer {
+ static Logger log = Logger.getLogger("ro.vivi.pistruiatul.VotingSession");
+
+ public static String VOTE_URL = "/pls/steno/evot.nominal?uniqueId={IDV}&idl=1";
+ public static String LAW_URL = "/pls/proiecte/upl_pck.proiect?idp={IDP}";
+
+ /** Extracts the deputy ID number and his name from the line with his name */
+ Pattern linkDeputyPattern = Pattern.compile(
+ "<td><A HREF=\"/pls/parlam/structura\\.mp\\?idm=(\\d*)&" +
+ "cam=(\\d)&leg=(\\d*)\">(.*)</A></td>");
+ /** Extracts the party of a deputy from his vote line */
+ Pattern partyPattern = Pattern.compile("<td align=\"center\">(.*)</td>");
+
+ Pattern lawPattern = Pattern.compile(
+ "<A HREF=\"/pls/proiecte/upl_pck\\.proiect\\?idp=([0-9]*)\" " +
+ "TARGET=\"PROIECTE\">PL (.*)</A>");
+
+ // <tr valign="top"><td align="right" bgcolor="#fffef2"
+ // nowrap>Subiect vot:</td><td><b>
+ String subjectLine =
+ "<tr valign=\"top\"><td align=\"right\" bgcolor=\"#fffef2\" nowrap>" +
+ "Subiect vot:</td><td><b>";
+
+ private int[] errorVotes = {5093, 5097};
+
+ private int[] miscVotes = {4887, 4898, 4915, 4985, 5046, 5056, 5065, 5098,
+ 5099, 5100, 5102, 5103, 5147, 5172, 5367, 5371, 5609, 5610, 5658};
+
+ // Whether we think this vote is an error or not.
+ public boolean error = false;
+
+ private String year;
+
+ /**
+ * Constructor.
+ * @param uniqueId The id of the vote, used to get to the link for that
+ * particular vote and fetch the list of deputies that voted for it.
+ */
+ public CdepNominalVote(long time, String uniqueId, String year) {
+ this.time = time;
+ this.uniqueId = uniqueId;
+ this.year = year;
+ }
+
+ /**
+ * Initiates the crawling of this particular vote.
+ */
+ public void run() {
+ String path = VOTE_URL.replace("{IDV}", uniqueId);
+ this.link = "http://" + Main.HOST + path;
+ log.info("Fetching vote " + path);
+ InternetsCrawler.enqueue(Main.HOST, path, this);
+ }
+
+
+ /**
+ * Consumes the page that comes from the server. Mainly parses the votes of
+ * the deputies, but also parses stuff like subject of the vote.
+ */
+ public void consume(String page) {
+ String[] lines = page.split("\n");
+ CdepLaw cdepLaw = null;
+
+ int i = 0;
+
+ log.info("Consuming " + VOTE_URL.replace("{IDV}", uniqueId));
+ while (i < lines.length) {
+ String line = lines[i].trim();
+
+ Matcher lawMatcher = lawPattern.matcher(line);
+ if (lawMatcher.matches()) {
+ String idp = lawMatcher.group(1);
+ String projectNumber = lawMatcher.group(2);
+
+ cdepLaw = Main.cdepLaws.getLaw(projectNumber, idp);
+ idLaw = cdepLaw.id;
+
+ log.info("This vote belongs to cdepLaw " + idLaw);
+ }
+
+ Matcher m = linkDeputyPattern.matcher(line);
+ if (m.matches()) {
+ int idm = Integer.parseInt(m.group(1));
+ String name = m.group(4);
+
+ Matcher pmatch = partyPattern.matcher(lines[i + 1]);
+ String party = pmatch.matches() ? pmatch.group(1) : "none";
+ String vote = lines[i + 3].trim();
+
+ Deputy dep = Main.deputies.getDeputyForIdm(idm);
+ if (dep == null) {
+ name = name.replace("-", " ");
+ dep = new Deputy(name, "" + idm);
+ Main.deputies.add(dep);
+ dep.getInfoFromSite();
+ }
+
+ dep.setParty(year, party, time);
+ dep.addVote(uniqueId, vote, time);
+
+ DbManager.insertPersonVote(Main.YEAR, link, vote, cdepLaw.id,
+ time, dep);
+ addSingleVote(new SingleVote(time, uniqueId, vote, dep));
+ }
+
+ // Test to see if somehow maybe this is the subject of the vote.
+ if (line.equals(subjectLine)) {
+ // Concatenate the next lines up to "</td>"
+ String desc = "";
+ int j = 0;
+
+ while (!lines[i + 1 + j].contains("</td>")) {
+ desc += lines[i + 1 + j++] + " ";
+ }
+ // The next line will be the description.
+ String separator = "<br>";
+ if (!desc.contains(separator)) {
+ separator = "- ";
+ }
+
+ if (desc.startsWith("Prezen")) {
+ // This is a presence vote.
+ type = desc;
+ subject = "";
+
+ } else if (desc.startsWith("Amendament")) {
+ type = "Amendament";
+ subject = desc;
+
+ } else if (!desc.contains(separator)) {
+ // This is in the type of "(type vote) Description vote.
+ Pattern p = Pattern.compile("[(](.*)[)] (.*)");
+ Matcher mdesc = p.matcher(desc);
+ if (mdesc.matches()) {
+ type = mdesc.group(1);
+ subject = mdesc.group(2);
+ } else {
+ // We manually mark the erroneous votes since they are hard to
+ // detect otherwise.
+ if (Arrays.binarySearch(errorVotes,
+ Integer.parseInt(uniqueId)) >= 0) {
+ this.error = true;
+ return;
+
+ } else if (Arrays.binarySearch(miscVotes,
+ Integer.parseInt(uniqueId)) >= 0) {
+ type = "misc";
+ subject = desc;
+ // This vote is ok, don't do anything.
+ } else {
+ log.warning("==== I believe this is an eroneous vote!");
+ // I think we should entirely ignore this vote.
+ System.exit(-1);
+ }
+ }
+ } else {
+ String parts[] = desc.split(separator);
+ type = parts[0].replace("</b>", "");
+ subject = parts[1];
+ }
+ i++;
+ }
+
+ i++;
+ }
+
+ if (type == null || subject == null) {
+ log.warning("-- This vote has no type or subject!");
+ } else {
+ log.info("Type:" + type + ", Subject:" + subject);
+ }
+ if (!error) {
+ DbManager.insertNominalVote("cdep", "2008", this);
+ }
+
+ log.info("Votes for vote id " + uniqueId + ", " +
+ getDateString(time) + " \n" +
+ "+ " + votesAgg[0] + " DA\n" +
+ "+ " + votesAgg[1] + " NU\n" +
+ "+ " + votesAgg[2] + " Abtinere\n" +
+ "+ " + votesAgg[3] + " -");
+ }
+
+
+ private String getDateString(long time) {
+ Date d = new Date();
+ d.setTime(time);
+ return new SimpleDateFormat("yyyy-MM-dd").format(d);
+ }
+}
View
168 java/src/ro/vivi/pistruiatul/CdepVotesPool.java
@@ -0,0 +1,168 @@
+package ro.vivi.pistruiatul;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data model for the cdepLaws that were voted electronically.
+ * @author vivi
+ */
+public class CdepVotesPool implements PageConsumer {
+ Logger log = Logger.getLogger("ro.vivi.pistruiatul.CdepVotesPool");
+
+ /** The url from which we will grab the daily votes. */
+ public static String SEED_PATH =
+ "/pls/steno/eVot.Data?dat={DATE}&cam=2&idl=1";
+
+ /** Keeping track of the cdepLaws, indexed by the id of the law. */
+ HashMap<String, VotingSession> votes = new HashMap<String, VotingSession>();
+
+ /** The current date I am parsing votes for. */
+ private Date currentDate;
+
+ private int daysWithVotes;
+ private int finalVotes;
+ private int totalVotes;
+
+ private String year;
+
+ /**
+ * Initializes some of the much needed data structures.
+ */
+ public CdepVotesPool() {
+ // noop.
+ }
+
+ /**
+ * Runs the parsing of the entire vote pool.
+ */
+ public void run(String year) {
+ this.year = year;
+
+ fetchVotesListFromSitePages();
+
+ // Go through each vote and parse it (parse representatives and stuff)
+ int count = 0;
+ for (String idv : votes.keySet()) {
+ count++;
+ log.info("Parsing " + count + "/" + votes.keySet().size());
+ CdepNominalVote nv = (CdepNominalVote) votes.get(idv);
+ nv.run();
+ }
+
+ log.info(" + Got deputies: " + Main.deputies.deps.size());
+
+ // For each deputy, print some stats. These stats should rather go to the
+ // database.
+ for (Deputy dep : Main.deputies.deps.values()) {
+ log.info(dep.toString() + "\n" +
+ dep.getVoteStatsString(votes.values()) + "\n" +
+ dep.getTimeInOfficeString());
+ dep.flushAggregateStatsToDb(votes.values(), year);
+ }
+ }
+
+ /**
+ * Fetches the list of cdepLaws from the site.
+ */
+ @SuppressWarnings("deprecation")
+ public void fetchVotesListFromSitePages() {
+ // Let's start with Nov 2008, even though we know that the first vote was on
+ // Feb 04 2009.
+ currentDate = new Date(108, 10, 30);
+ Date now = new Date();
+
+ // For testing, we might need to only parse a few days.
+ int count = Integer.MAX_VALUE;
+
+ while (now.getTime() > currentDate.getTime() && count-- > 0) {
+ currentDate.setDate(currentDate.getDate() + 1);
+ String path = SEED_PATH.replace("{DATE}", "" +
+ getDateString(currentDate));
+
+ log.info("Fetching page: " + path);
+ InternetsCrawler.enqueue(Main.HOST, path, this);
+ }
+ log.info(" -- Done! " + daysWithVotes + " days with votes, " + finalVotes +
+ " final votes, " + totalVotes + " total votes.");
+ }
+
+ private String getDateString(Date d) {
+ return new SimpleDateFormat("yyyyMMdd").format(d);
+ }
+
+ /**
+ * @inheritDoc
+ */
+ @SuppressWarnings("deprecation")
+ public void consume(String page) {
+ //<A HREF="/pls/steno/evot.nominal?uniqueId=5153&idl=1">deschis</A>
+ Pattern dateAndTime = Pattern.compile("<A HREF=\"/pls/steno/evot\\." +
+ "nominal\\?idv=([0-9]+)&idl=([0-9]+)\">(.*)</A>");
+
+ // consume the page that came back.
+ String[] lines = page.split("\n");
+ int i = 0;
+
+ boolean hasAny = false;
+
+ while (i < lines.length) {
+ String line = lines[i].trim();
+ Matcher m = dateAndTime.matcher(line);
+ if (m.matches()) {
+ // We have a law, let's create an object
+ String idv = m.group(1);
+ String timeString = m.group(3);
+
+ if (timeString.indexOf(":") >= 0) {
+ String[] hm = timeString.split(":");
+ currentDate.setHours(Integer.parseInt(hm[0]));
+ currentDate.setMinutes(Integer.parseInt(hm[1]));
+ } else {
+ currentDate.setHours(0);
+ currentDate.setMinutes(0);
+ }
+ long time = currentDate.getTime();
+
+ // Now I might have a line with the type which can be Adoptare,
+ // Respingere If it doesn't start with <A HREF
+ // This changes in time, maybe we shouldn't rely on it?
+ i += 6;
+ String type = lines[i];
+
+ if (type.startsWith("Vot final") || type.indexOf("vot final") > 0) {
+ finalVotes++;
+ }
+ totalVotes++;
+
+ // Create a new VotingSession object that will go ahead and parse the
+ // information about the vote, who was there and who wasn't.
+ CdepNominalVote nv = new CdepNominalVote(time, idv, year);
+ votes.put(idv, nv);
+
+ log.info(" + vote parsed from list time=" + time + " uniqueId=" + idv +
+ " type=" + type);
+ hasAny = true;
+ }
+ i++;
+ }
+ if (hasAny) {
+ daysWithVotes++;
+ }
+ }
+
+ /**
+ * Returns a reference to the pool of deputies.
+ */
+ public Deputies getDeputies() {
+ return Main.deputies;
+ }
+
+ public HashMap<String, VotingSession> getVotes() {
+ return votes;
+ }
+}
View
839 java/src/ro/vivi/pistruiatul/DbManager.java
@@ -0,0 +1,839 @@
+package ro.vivi.pistruiatul;
+
+import java.sql.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Manages the connection with the database, knows how to insert stuff in there
+ * and also knows how to read stuff from there.
+ * @author vivi
+ */
+public class DbManager {
+ static Logger log = Logger.getLogger("ro.vivi.pistruiatul.DbManager");
+
+ static boolean NOOP = false;
+
+ public static Connection conn;
+ public static PreparedStatement deletePartyPs = null;
+ public static PreparedStatement insertPartyPs = null;
+ public static PreparedStatement deleteDeputyVotePs = null;
+ public static PreparedStatement insertDeputyVotePs = null;
+ public static PreparedStatement updateMaverickVotePs = null;
+ /** Initializes the connection with the database. */
+ static {
+ try {
+ Class.forName("com.mysql.jdbc.Driver").newInstance();
+ } catch (Exception ex) { ex.printStackTrace(); }
+
+ try {
+ conn = DriverManager.getConnection("jdbc:mysql://localhost/" +
+ "hartapoliticii_pistruiatul?user=root&password=root&" +
+ "characterEncoding=utf-8");
+
+ } catch (SQLException ex) {
+ // handle any errors
+ System.out.println("SQLException: " + ex.getMessage());
+ System.out.println("SQLState: " + ex.getSQLState());
+ System.out.println("VendorError: " + ex.getErrorCode());
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadParliamentParties(HashMap<String, Integer> parties) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery("SELECT id, name FROM parties");
+ while (rs.next()) {
+ int id = rs.getInt(1);
+ String name = rs.getString(2);
+
+ parties.put(name, new Integer(id));
+ }
+ rs.close();
+ stmt.close();
+
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadElectionsParties(HashMap<String, Integer> parties) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery("SELECT id, name, long_name " +
+ "FROM parties group by name");
+ while (rs.next()) {
+ int id = rs.getInt(1);
+ String name = Utils.replaceDiacritics(
+ rs.getString(3).trim().toLowerCase());
+
+ parties.put(name, new Integer(id));
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadCdepLaws(CdepLaws laws, String year) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT id, link, number, text FROM cdep_" + year + "_laws");
+ while (rs.next()) {
+ int id = rs.getInt(1);
+ String link = rs.getString(2);
+ String number = rs.getString(3);
+ String text = rs.getString(4);
+
+ CdepLaw cdepLaw = new CdepLaw(number, laws);
+ cdepLaw.id = id;
+
+ // http://www.cdep.ro/pls/proiecte/upl_pck.proiect?idp=123
+ int pos = link.indexOf("idp=");
+ cdepLaw.idp = link.substring(pos + "idp=".length());
+
+ cdepLaw.description = text;
+ laws.laws.put(number, cdepLaw);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+
+ /** Loads the deputies that are currently in the data base */
+ public static void getSeatsForCounty(SeatGroup sg, List<Seat> seats) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT colegiu FROM results_2008 where colegiu like \"" +
+ sg.room + "% " + sg.county + "\" group by colegiu");
+ while (rs.next()) {
+ Seat s = new Seat();
+ s.name = rs.getString(1).trim();
+
+ seats.add(s);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void getAllocatedSeatsPerParty(SeatGroup sg,
+ HashMap<String, Integer> allocated) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT partid, numar, idpartid FROM results_2008_allocated " +
+ "where judet =\"" + sg.county + "\" and room = \"" +
+ sg.room + "\"");
+ while (rs.next()) {
+ String party = Utils.replaceDiacritics(
+ rs.getString(1).trim().toLowerCase());
+ Integer seats = new Integer(rs.getInt(2));
+ allocated.put(party, seats);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+
+ /** Loads the deputies that are currently in the data base */
+ public static void getCandidatesForSeatGroup(SeatGroup sg,
+ List<Candidate> candidates) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT nume, colegiu, voturi, partid, idcandidat, idperson " +
+ "FROM results_2008 " +
+ "where colegiu like \"" + sg.room + "% " + sg.county + "\"");
+ while (rs.next()) {
+ Candidate c = new Candidate();
+ c.name = rs.getString(1);
+ c.runsForSeat = sg.getSeat(rs.getString(2).trim());
+ c.votes = rs.getInt(3);
+ c.partyName = Utils.replaceDiacritics(
+ rs.getString(4).trim().toLowerCase());
+ c.party = ElectionsParties.getPartyId(c.partyName);
+ c.id = rs.getInt(5);
+ c.idperson = rs.getInt(6);
+ candidates.add(c);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the senate laws */
+ public static void loadSenateLaws(SenateLaws laws, String year) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT id, appid FROM senat_" + year + "_laws");
+ while (rs.next()) {
+ int id = rs.getInt(1);
+ String appId = rs.getString(2);
+
+ SenateLaw law = new SenateLaw(laws);
+ law.id = id;
+ law.appId = appId;
+ laws.laws.put(appId, law);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadDeputies(HashMap<String, Deputy> deps, String year) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT id, idm, name, timein, timeout " +
+ "FROM cdep_" + year + "_deputies");
+ while (rs.next()) {
+ String idm = "" + rs.getInt(2);
+ String name = rs.getString(3);
+
+ Deputy dep = new Deputy(name, idm);
+ dep.id = "" + rs.getInt(1);
+
+ dep.startTime = rs.getLong(4);
+ if (rs.getLong(5) > 0) {
+ dep.endTime = rs.getLong(5);
+ }
+
+ deps.put(name, dep);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadSeatGroups(HashMap<String, SeatGroup> seatGroups) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery("SELECT judet, room FROM " +
+ "results_2008_allocated group by judet, room");
+ while (rs.next()) {
+ String room = rs.getString(2);
+ String county = rs.getString(1);
+ seatGroups.put(room + county, new SeatGroup(county, room));
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Loads the deputies that are currently in the data base */
+ public static void loadSenators(String year,
+ HashMap<String, Senator> senators) {
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(
+ "SELECT name, id, idperson " +
+ "FROM senat_" + year + "_senators");
+ while (rs.next()) {
+ String name = rs.getString(1);
+ int id = rs.getInt(2);
+
+ // HACK: for now we don't have the senator's real id. This would be the
+ // id from the cdep site, but we don't have that yet. Will have to
+ // refactor this I guess.
+ Senator sen = new Senator(name, "" + id);
+ sen.id = "" + id;
+
+ senators.put(name, sen);
+ }
+ rs.close();
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertSeatInfoToAggregates(SeatGroup sg, Seat seat) {
+ String collegeNr = seat.name.substring(0, seat.name.indexOf(' '));
+ String sql =
+ "INSERT INTO results_2008_agg(college, winnerid, idperson_winner, " +
+ "runnerupid, idperson_runnerup, reason, " +
+ "total, difference, winvotes, runvotes, " +
+ "county, college_nr) " +
+ "values('" + seat.name + "', " +
+ seat.winner.id + ", " + seat.winner.idperson + ", " +
+ seat.runnerUp.id + ", " + seat.runnerUp.idperson + ", " + "'" +
+ seat.runnerUpReason + "', " + seat.totalVotes + ", " +
+ seat.runnerUpVotes + ", " + seat.winner.votes + ", " +
+ seat.runnerUp.votes + ", '" + sg.county + "', '" +
+ collegeNr + "')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertDeputy(Deputy dep) {
+ String sql =
+ "INSERT IGNORE INTO cdep_" + Main.YEAR +
+ "_deputies (idm, name, timein) " +
+ "values (" + dep.cdepId + ", '" + dep.name + "', " +
+ dep.startTime + ")";
+ int result = updateWithAutoIncrement(sql);
+
+ dep.idperson = getIdPersonForDeputy("2008", dep);
+ return result;
+ }
+
+ /**
+ * Fetches the person id for a certain deputy, given the year in which this
+ * guy is expected to have been a deputy.
+ * @param year
+ * @return
+ */
+ public static int getIdPersonForDeputy(String year, Deputy dep) {
+ String sql =
+ "SELECT idperson FROM cdep_" + year + "_deputies " +
+ "WHERE idm=" + dep.cdepId;
+ return selectInt(sql);
+ }
+
+ /**
+ * Fetches the person id for a certain deputy, given the year in which this
+ * guy is expected to have been a deputy.
+ * @param year
+ * @return
+ */
+ public static int getIdPersonForSenator(String year, Senator sen) {
+ String sql =
+ "SELECT idperson FROM senat_" + year + "_senators " +
+ "WHERE name='" + sen.name + "'";
+ return selectInt(sql);
+ }
+
+ /** Inserts a senator in the database */
+ public static int insertSenator(Senator sen) {
+ String sql =
+ "INSERT INTO senat_2004_senators (name, idm) " +
+ "values ('" + sen.name + "', '" + sen.idm + "')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertCdepLaw(CdepLaw cdepLaw) {
+ String sql =
+ "INSERT INTO cdep_" + Main.YEAR +
+ "_laws(link, number, text) " +
+ "values('" + cdepLaw.link + "', '" + cdepLaw.projectNumber + "', '" +
+ cdepLaw.description + "') ";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertLaw(int room, String year, String lawLink,
+ String projectNumber, String desc) {
+ String roomName = "";
+ switch(room) {
+ case Main.SENAT: roomName = "senat"; break;
+ case Main.CDEP: roomName = "cdep"; break;
+ }
+ // Insert the law.
+ updateWithAutoIncrement(
+ "INSERT IGNORE INTO " + roomName + "_" + year +
+ "_laws(link, number, text) " +
+ "values('" + lawLink + "', '" + projectNumber + "', '" + desc + "') ");
+
+ // Because the ID doesn't seem to work correctly when we insert ignore, we
+ // just look it up again.
+ return selectInt("SELECT id FROM " + roomName + "_" + year + "_laws " +
+ "WHERE link='" + lawLink + "'");
+ }
+
+ /** Inserts a deputy in the database */
+ public static void updateCdepLawDescription(int id, String text) {
+ String sql =
+ "update cdep_" + Main.YEAR + "_laws SET " +
+ "text='" + text + "' " +
+ "WHERE id=" + id;
+ update(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertSenateLaw(SenateLaw law) {
+ String sql =
+ "INSERT INTO senat_2004_laws(appId) " +
+ "values('" + law.appId + "')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertParty(String name) {
+ String sql =
+ "INSERT INTO parties(name) " +
+ "values('" + name + "')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertElectionsParty(String name) {
+ String sql =
+ "INSERT INTO results_2008_parties(name) " +
+ "values('" + name + "')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /**
+ * Inserts a deputy in the database.
+ */
+ public static void insertPersonVote(
+ String year, String voteLink, String vote,
+ int lawId, long time, Deputy dep) {
+ if (NOOP) return;
+ try {
+ if (deleteDeputyVotePs == null) {
+ deleteDeputyVotePs = conn.prepareStatement(
+ "DELETE FROM " + dep.getRoom() + "_" + year + "_votes " +
+ "WHERE idperson= ? AND link= ?");
+ }
+ deleteDeputyVotePs.setInt(1, dep.idperson);
+ deleteDeputyVotePs.setString(2, voteLink);
+ deleteDeputyVotePs.executeUpdate();
+
+ if (insertDeputyVotePs == null) {
+ String idField = dep.getRoom().equals("cdep") ? "iddep" : "idsen";
+ insertDeputyVotePs = conn.prepareStatement(
+ "INSERT INTO " + dep.getRoom() + "_" + year + "_votes" +
+ "(link, idlaw, vote, time, " + idField + ", idperson) " +
+ "VALUES(?, ?, ?, ?, ?, ?)");
+ }
+ insertDeputyVotePs.setString(1, voteLink);
+ insertDeputyVotePs.setInt(2, lawId);
+ insertDeputyVotePs.setString(3, vote);
+ insertDeputyVotePs.setLong(4, time);
+ insertDeputyVotePs.setInt(5, Integer.parseInt(dep.id));
+ insertDeputyVotePs.setInt(6, dep.idperson);
+ insertDeputyVotePs.executeUpdate();
+
+ } catch (SQLException se) {
+ System.out.println("Did not delete " + voteLink + " " + time + " " +
+ dep.idperson);
+ se.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void updateVoteIsMaverick(Deputy deputy, VotingSession session,
+ SingleVote vote, String year) {
+ if (NOOP) return;
+ try {
+ if (updateMaverickVotePs == null) {
+ updateMaverickVotePs = conn.prepareStatement(
+ "UPDATE " + deputy.getRoom() + "_" + year + "_votes " +
+ "SET maverick=1 " +
+ "WHERE link= ? AND time= ? AND idperson= ?");
+ }
+ updateMaverickVotePs.setString(1, session.link);
+ updateMaverickVotePs.setLong(2, vote.time);
+ updateMaverickVotePs.setInt(3, deputy.idperson);
+ updateMaverickVotePs.executeUpdate();
+
+ } catch (SQLException se) {
+ se.printStackTrace();
+ log.severe(updateMaverickVotePs.toString());
+ System.exit(1);
+ }
+ }
+
+ /**
+ * Inserts a vote - number, description, type, time - into the database.
+ */
+ public static void insertNominalVote(String what, String year,
+ VotingSession v) {
+ // replace the old data.
+ update("DELETE FROM " + what + "_" + year + "_votes_details WHERE " +
+ "link = '" + v.link + "'");
+
+ String sql =
+ "INSERT INTO " + what + "_" + year + "_votes_details " +
+ " (link, idlaw, vda, vnu, vab, vmi, type, description, time) " +
+ "VALUES('" +
+ v.link + "', " +
+ v.idLaw + ", " +
+ v.votesAgg[0] + ", " +
+ v.votesAgg[1] + ", " +
+ v.votesAgg[2] + ", " +
+ v.votesAgg[3] + ", '" +
+ v.type + "', '" +
+ v.subject + "', " +
+ v.time +")";
+ //System.out.println(sql);
+ update(sql);
+ }
+
+ /**
+ * Adds to the database the aggregate stats about this deputy. The previous
+ * stats are simply removed and replaced.
+ */
+ public static void insertAggregateDeputyStats(String what, Deputy dep,
+ String year) {
+ // Clear the previous aggregates.
+ update("DELETE FROM " + what + "_" + year + "_votes_agg WHERE idperson=" +
+ dep.idperson);
+
+ // Insert the new aggregates.
+ String sql =
+ "INSERT INTO " +
+ what + "_" + year + "_votes_agg(iddep, idperson, vda, vnu, vab, " +
+ "vmi, possible, percent, maverick, days_in, days_possible) " +
+ "VALUES(" + dep.idm + ", " + dep.idperson + ", " +
+ dep.getVotes(SingleVote.Type.DA) + ", " +
+ dep.getVotes(SingleVote.Type.NU) + ", " +
+ dep.getVotes(SingleVote.Type.AB) + ", " +
+ dep.getVotes(SingleVote.Type.MI) + ", " +
+ dep.getPossibleVotes() + ", " +
+ dep.getVotesPercent() + ", " +
+ dep.getMaverickPercent() + ", " +
+ dep.getWorkDays() + ", " +
+ dep.getPossibleDays() + ")";
+
+ //System.out.println(sql);
+ update(sql);
+ }
+
+ /**
+ * Inserts a fact about a particular party.
+ * @param partyId The id of the party for which we are adding a fact.
+ * @param attr The attribute, the name of the fact.
+ * @param value The value.
+ */
+ public static void insertPartyFact(int partyId, String attr, String value) {
+ long time = System.currentTimeMillis();
+ String sql =
+ "INSERT IGNORE INTO parties_facts(idparty, attribute, value, time_ms) " +
+ "values(" + partyId + ", '" + attr + "', '" + value + "', " + time + ")";
+ update(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertSenatorVote(String appId, String vote, Senator sen,
+ SenateLaw law, long time) {
+ String sql =
+ "INSERT INTO senat_2004_votes(uniqueId, idlaw, idsen, vote, time) " +
+ "values('" + appId + "', " + law.id + ", " + sen.idm + ", '" +
+ vote + "', " + time + ")";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertVideo(Deputy dep, String idv, int sessions,
+ int seconds) {
+ String sql =
+ "INSERT INTO " + Main.ROOM + "_" + Main.YEAR +
+ "_video(uniqueId, iddep, sessions, seconds) " +
+ "values(" + idv + ", " + dep.idm + ", " + sessions + ", " + seconds + ")";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertLawProponent(int idlaw, int iddep, int chamber,
+ int authorscount) {
+ String sql =
+ "INSERT IGNORE INTO " + Main.ROOM + "_" + Main.YEAR +
+ "_laws_proponents(idlaw, iddep, chamber, authorscount) " +
+ "values(" + idlaw + ", " + iddep + ", " + chamber + ", " +
+ authorscount + ")";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertAwayTime(int idperson, int chamber, long left,
+ long back) {
+ String sql =
+ "INSERT INTO away_times(idperson, chamber, time_left, time_back," +
+ "reason) " +
+ "values(" + idperson + ", " + chamber + ", " + left + ", " +
+ back + ", 'EuroParlamentar')";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertLawStatus(int idlaw, int status) {
+ String sql =
+ "INSERT IGNORE INTO " + Main.ROOM + "_" + Main.YEAR +
+ "_laws_status(idlaw, status) " +
+ "values(" + idlaw + ", " + status + ")";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static void insertPartyBelonging(String what, String year,
+ String depid, int idperson,
+ int partyid, long time) {
+ if (NOOP) return;
+ try {
+ if (deletePartyPs == null) {
+ deletePartyPs = conn.prepareStatement(
+ "DELETE FROM " + what + "_" + year + "_belong " +
+ "WHERE idperson= ? AND time= ? ");
+ }
+ deletePartyPs.setInt(1, idperson);
+ deletePartyPs.setLong(2, time);
+ deletePartyPs.executeUpdate();
+
+ if (insertPartyPs == null) {
+ insertPartyPs = conn.prepareStatement(
+ "INSERT INTO " + what + "_" + year + "_belong " +
+ "(iddep, idperson, idparty, time) VALUES(?, ?, ?, ?)");
+ }
+ insertPartyPs.setInt(1, Integer.parseInt(depid));
+ insertPartyPs.setInt(2, idperson);
+ insertPartyPs.setInt(3, partyid);
+ insertPartyPs.setLong(4, time);
+ insertPartyPs.executeUpdate();
+
+ } catch (SQLException se) {
+ se.printStackTrace();
+
+ log.info(deletePartyPs.toString());
+ log.info(insertPartyPs.toString());
+
+ System.exit(1);
+ }
+ }
+
+ /** Inserts a deputy in the database */
+ public static void insertCurrentBelonging(String what, String year,
+ Deputy dep) {
+ // Clear the previous aggregates.
+ update("DELETE FROM " + what + "_" + year + "_belong_agg WHERE idperson=" +
+ dep.idperson);
+ String sql =
+ "INSERT INTO " + what + "_" + year + "_belong_agg" +
+ "(iddep, idperson, idparty) " +
+ "VALUES(" + dep.cdepId + ", " + dep.idperson + ", " +
+ dep.mostRecentPartyId + ") ";
+ update(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertSenatorPartyBelonging(String year, int id, int pid,
+ long time) {
+ String sql =
+ "INSERT INTO senat_" + year + "_belong(idsen, idparty, time) " +
+ "values(" + id + ", " + pid + ", " + time + ") ";
+
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertEuroCandidate2009(Person p) {
+ String sql =
+ "INSERT INTO euro_2009_candidates(name, profession, occupation, " +
+ "birthday, idparty, position, idperson) " +
+ "values('" + p.name + "', '" + p.getFact("profession") + "', '" +
+ p.getFact("occupation") + "', " + p.getFact("birthday") + ", " +
+ p.getFact("idparty") + ", " + p.getFact("position") + ", 0)";
+ return updateWithAutoIncrement(sql);
+ }
+
+
+ /** Inserts a deputy in the database */
+ public static void updateDeputy(long startDate, long endDate,
+ String motif, int id) {
+ String sql =
+ "UPDATE cdep_" + Main.YEAR + "_deputies SET " +
+ "timein = '" + startDate + "', " +
+ "timeout = '" + endDate + "', " +
+ "motif = '" + motif + "' " +
+ "where cdep_" + Main.YEAR + "_deputies.id = " + id;
+ update(sql);
+ }
+
+ /** Update the image of a deputy */
+ public static void updateDeputyImage(Deputy dep, String img) {
+ String sql =
+ "UPDATE cdep_" + Main.YEAR + "_deputies SET imgurl = '" + img + "' " +
+ "where cdep_" + Main.YEAR + "_deputies.idm = " + dep.idm;
+ update(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static void updateSenator(long startDate, long endDate,
+ String motif, int id) {
+ String sql =
+ "UPDATE senat_2004_senators SET timein = '" + startDate + "', " +
+ "timeout = '" + endDate + "', " +
+ "motif = '" + motif + "' " +
+ "where senat_2004_senators.id = " + id;
+ update(sql);
+ }
+
+ /** Update the image of a deputy */
+ public static void updateSenatorImage(Senator sen, String img) {
+ String sql =
+ "UPDATE senat_2004_senators SET imgurl = '" + img + "' " +
+ "where senat_2004_senators.idm = " + sen.idm;
+ update(sql);
+ }
+
+
+ public static void updateCandidateReason(Candidate candidate, int winner,
+ int neededVotes, String reason) {
+ String sql =
+ "UPDATE results_2008_candidates " +
+ "SET winner=" + winner + ", " +
+ "college='" + candidate.runsForSeat.name + "', " +
+ "difference=" + neededVotes + ", " +
+ "reason='" + reason + "' " +
+ "WHERE id = " + candidate.id;
+ update(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertCatavencu(String name, String text, String url,
+ String party) {
+ String sql =
+ "INSERT INTO catavencu_2008(name, t, url, party, idperson) " +
+ "values('" + name + "', '" + text + "', '" + url + "', " +
+ "'" + party + "', 0)";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertEuroParliamentPresence(String name, long time) {
+ String sql =
+ "INSERT INTO euro_parliament_2007(name, time) " +
+ "values('" + name + "', " + time + ")";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Inserts a deputy in the database */
+ public static int insertQvorumEntry(String name, String text) {
+ String sql =
+ "INSERT INTO euro_parliament_2007_qvorum(name, text, idperson) " +
+ "values('" + name + "', '" + text + "', 0)";
+ return updateWithAutoIncrement(sql);
+ }
+
+ /** Run a generic update statement. Dumb java and jconnector. */
+ private static int updateWithAutoIncrement(String update) {
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(update);
+
+ int autoIncKeyFromFunc = -1;
+ ResultSet rs = stmt.executeQuery("SELECT LAST_INSERT_ID()");
+ if (rs.next()) {
+ autoIncKeyFromFunc = rs.getInt(1);
+ }
+ rs.close();
+ stmt.close();
+ return autoIncKeyFromFunc;
+
+ } catch (SQLException se) {
+ System.out.println(update);
+ se.printStackTrace();
+ }
+ return -1;
+ }
+
+ private static int getLastInsertId(Statement s) throws SQLException {
+ int autoIncKeyFromFunc = -1;
+ ResultSet rs = s.executeQuery("SELECT LAST_INSERT_ID()");
+ if (rs.next()) {
+ autoIncKeyFromFunc = rs.getInt(1);
+ }
+ rs.close();
+ return autoIncKeyFromFunc;
+ }
+
+ /** Delete all the votes */
+ public static void deleteAllFromVotes() {
+ update("TRUNCATE TABLE votes");
+ update("TRUNCATE TABLE belongs");
+ }
+
+ /** Delete all the votes */
+ public static void emptyResults2008Aggregates() {
+ update("TRUNCATE TABLE results_2008_agg");
+ }
+
+
+ /** Delete all the votes */
+ public static void deleteAllFromSenateVotes() {
+ update("TRUNCATE TABLE votes_senators");
+ update("TRUNCATE TABLE senators_belongs");
+ }
+
+ /** Delete all the votes */
+ public static void deleteEuro2009() {
+ update("TRUNCATE TABLE euro_2009_candidates");
+ }
+
+ /** Delete all the votes */
+ public static void deleteQvorum2007() {
+ update("TRUNCATE TABLE euro_parliament_2007_qvorum");
+ }
+
+ /** Delete all the votes */
+ public static void deleteCatavencu() {
+ update("TRUNCATE TABLE catavencu_2008");
+ }
+
+ /** Delete all the votes */
+ public static void deleteEuroParliament() {
+ update("TRUNCATE TABLE euro_parliament_2007");
+ }
+
+ /**
+ * Run a generic update statement. Dumb java and jconnector.
+ * @param update The sql query to be executed.
+ */
+ private static void update(String update) {
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(update);
+ stmt.close();
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ }
+
+ private static int selectInt(String select) {
+ int result = -1;
+ try {
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(select);
+ while (rs.next()) {
+ result = rs.getInt(1);
+ }
+ rs.close();
+ stmt.close();
+
+ } catch (SQLException se) {
+ se.printStackTrace();
+ }
+ return result;
+ }
+}
View
111 java/src/ro/vivi/pistruiatul/Deputies.java
@@ -0,0 +1,111 @@
+package ro.vivi.pistruiatul;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.util.HashMap;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Holds, parses and does everything that is attached with the list of deputies.
+ * It will usually just load the list from our own file / database.
+ * @author vivi
+ *
+ */
+public class Deputies {
+ Logger log = Logger.getLogger("ro.vivi.pistruiatul.Deputies");
+
+ Pattern percent =
+ Pattern.compile("(.*)votes missed</a> \\((.*)%\\), <a href=(.*)");
+
+ /** A repository of deputies */
+ public HashMap<String, Deputy> deps = new HashMap<String, Deputy>();
+
+ /**
+ * The url of a random law containing an initial set of deputies from which
+ * we will start making a list of each of them. Then, once we have.
+ */
+ public static String SEED_LAW_URL =
+ "http://www.cdep.ro/pls/steno/eVot.Nominal?uniqueId=502";
+
+ /**
+ * Read the currently existing list of deputies from the database.
+ */
+ public Deputies(String year) {
+ loadFromDb(year);
+ }
+
+ /** Loads the deputies from the database */
+ private void loadFromDb(String year) {
+ DbManager.loadDeputies(deps, year);
+ log.info("Deputies loaded from db: " + deps.size());
+ }
+
+ /**
+ * Reads more info about the deputies and updates the database with it.
+ */
+ public void enhanceInfo() {
+ for (Deputy dep : deps.values()) {
+ dep.getInfoFromSite();
+ // here I should update the database.
+ }
+ }
+
+ public Deputy getDeputyForIdm(int idm) {
+ for (Deputy dep : deps.values()) {
+ if (dep.idm == idm) {
+ return dep;
+ }
+ }
+ return null;
+ }
+
+ public int getIdForIdm(int idm) {
+ for (Deputy deputy : deps.values()) {
+ if (deputy.idm == idm) {
+ return deputy.idm;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Checks whether this deputy is already in the chamber of deputies.
+ * @param name
+ * @return
+ */
+ public boolean contains(String name) {
+ return deps.containsKey(name);
+ }
+
+ /**
+ * Returns a deputy with this exact name.
+ * @param name The exact name of the deputy.
+ * @return The deputy object, or null if this deputy does not exist.
+ */
+ public Deputy get(String name) {
+ return deps.get(name);
+ }
+
+ /**
+ * Add a deputy to the data model.
+ */
+ public void add(Deputy deputy) {
+ deps.put(deputy.name, deputy);
+
+ // Add this deputy to mysql.
+ storeInMysql(deputy);
+ }
+
+ /**
+ * Given a deputy object, it stores it in mysql. After that, it stores the
+ * mysql id in the object too (so that when a vote or whatever else needs to
+ * reference this, it just knows).
+ * @param deputy
+ */
+ public void storeInMysql(Deputy deputy) {
+ DbManager.insertDeputy(deputy);
+ }
+
+}
View
595 java/src/ro/vivi/pistruiatul/Deputy.java
@@ -0,0 +1,595 @@
+package ro.vivi.pistruiatul;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Mostly a data structure with information about a deputy. This will be used
+ * as a data model, more or less an image of what is in the database.
+ *
+ * @author vivi
+ */
+public class Deputy implements PageConsumer {
+ static Logger log = Logger.getLogger("ro.vivi.pistruiatul.Deputy");
+
+ Pattern datePattern =
+ Pattern.compile("([\\d]*)([ ]*)([\\w]*)([\\s]*)([\\d]*)");
+
+ String[] months = {"ianuarie", "februarie", "martie", "aprilie", "mai",
+ "iunie", "iulie", "august", "septembrie", "octombrie",
+ "noiembrie", "decembrie"};
+
+ Pattern moreInfoPattern =
+ Pattern.compile("(.*)<br>data validarii: (.*)( - " +
+ "<a href=([^>]*)>([^>])*</a>)?" +
+ "(<br>data �ncetarii mandatului:([^>]*) - (\\w*))?" +
+ "(<br>�nlocuit de: <([^>]*)><b>([\\w ]*)</b></A>)?" +
+ "(<br>�nlocuieste pe: <([^>]*)><b>([\\w ]*)</b></A>)?" +
+ "</td>");
+ // data validarii: 17 februarie 2004 -
+ // <a href="/pls/legis/legis_pck.htp_act?ida=54223" target="LEGIS">
+ // HS nr.57/2004</a>
+ Pattern dataValidariiP = Pattern.compile("data validarii: ([^>-]*)( - " +
+ "(<a href=([^>]*)>)?([^>])*(</a>)?)?(</td>)?");
+ Pattern dataIncetariiP = Pattern.compile("data �ncetarii mandatului:([^>]*)" +
+ " - (\\w*)( - [^-]*)?(</td>)?");
+ Pattern inlocuitDeP =
+ Pattern.compile("�nlocuit de: <([^>]*)><b>([\\w ]*)</b></A>(</td>)?");
+ Pattern inlocuiesteP =
+ Pattern.compile("�nlocuieste pe: <([^>]*)><b>([\\w ]*)</b></A>(</td>)?");
+
+ /** Fetches the url for talking time and list of video footage. */
+ Pattern talkTimeInfo =
+ Pattern.compile("(.*)ri de cuv�nt �n plen: </td><td>" +
+ "<A HREF=\"(/pls/steno/steno.lista\\?idv=(\\d*)&leg=2004&" +
+ "idl=(\\d*))\"><b>(\\d*)</b>(.*)");
+
+ protected static String DEP_URL =
+ "pls/parlam/structura.mp?idm={IDM}&cam=2&leg=2008";
+
+ /** The deputy's name */
+ public String name;
+
+ /** The deputy id's in the cdep.ro site. */
+ public String cdepId;
+ public int idm;
+
+ /**
+ * The id of the senator in the senators table. This is not the person
+ * id. If we ever need the person ID, we just get it straight from the
+ * database through a helper method.
+ */
+ public String id;
+
+ /** The id of the most recent party this candidate belongs to. */
+ public int mostRecentPartyId;
+
+ /** The time of the most recent party belonging data. */
+ private long mostRecentPartyTime = 0;
+
+ /**
+ * We make an object for each video list of each candidate. This makes it
+ * easier to have that be a PageConsumer.
+ */
+ private Video video;
+
+ /**
+ * When his time in office started. This is usually election day, but for
+ * people that quite or died or whatever, new people replace them and they
+ * have a new starting date.
+ */
+ public long startTime;
+
+ /** When his time in office ended. */
+ public long endTime;
+
+ /** The number of votes that this candidate could have voted on. */
+ public int possibleVotes;
+
+ /** The number of days in which there were any types of votes. */
+ public int possibleDays;
+
+ /**
+ * The person id of this deputy. This field is initialized at construction
+ * time if it's available, or else at the time of insertion in the database.
+ */
+ public int idperson;
+
+ /**
+ * Stores the votes of this deputy during his term. On this data model we will
+ * be able to compute his stats about presence.
+ *
+ * NOTE: we should also be able to use this for the more complicated
+ * statistics, in which we align his votes with other people's votes. So we
+ * should make this data structure flexible enough.
+ */
+ private HashMap<String, SingleVote> myVotes = new HashMap<String, SingleVote>();
+
+ private HashMap<SingleVote.Type, Integer> votesAgg =
+ new HashMap<SingleVote.Type, Integer>();
+
+ private int partyLineVotes = 0;
+ private int maverickVotes = 0