From 67de52da0d1589d1fb5f5486a028cff89f28fb97 Mon Sep 17 00:00:00 2001 From: brl Date: Sat, 16 Nov 2013 21:29:42 -0500 Subject: [PATCH] Refactor of DirectoryDownloader --- .../subgraph/orchid/DirectoryDownloader.java | 22 ++- src/com/subgraph/orchid/Tor.java | 4 +- src/com/subgraph/orchid/TorClient.java | 2 +- .../orchid/circuits/guards/Bridges.java | 19 ++- .../circuits/hs/HSDescriptorDownloader.java | 7 +- .../AbstractDirectoryDownloadTask.java | 116 ---------------- .../BridgeDescriptorDownloadTask.java | 78 ----------- .../downloader/BridgeDescriptorFetcher.java | 19 +++ .../downloader/CertificateDownloadTask.java | 66 --------- .../downloader/CertificateFetcher.java | 40 ++++++ .../downloader/ConsensusDownloadTask.java | 61 --------- .../downloader/ConsensusFetcher.java | 29 ++++ .../downloader/DescriptorDownloadTask.java | 110 --------------- .../DirectoryDocumentRequestor.java | 125 ++++++++++++++++++ .../downloader/DirectoryDownloadTask.java | 101 ++++++++------ .../downloader/DirectoryDownloaderImpl.java | 112 ++++++++++++---- .../DirectoryRequestFailedException.java | 15 +++ .../directory/downloader/DocumentFetcher.java | 52 ++++++++ .../directory/downloader/HttpConnection.java | 36 +++-- .../downloader/MicrodescriptorFetcher.java | 44 ++++++ .../downloader/RouterDescriptorFetcher.java | 43 ++++++ 21 files changed, 581 insertions(+), 520 deletions(-) delete mode 100644 src/com/subgraph/orchid/directory/downloader/AbstractDirectoryDownloadTask.java delete mode 100644 src/com/subgraph/orchid/directory/downloader/BridgeDescriptorDownloadTask.java create mode 100644 src/com/subgraph/orchid/directory/downloader/BridgeDescriptorFetcher.java delete mode 100644 src/com/subgraph/orchid/directory/downloader/CertificateDownloadTask.java create mode 100644 src/com/subgraph/orchid/directory/downloader/CertificateFetcher.java delete mode 100644 src/com/subgraph/orchid/directory/downloader/ConsensusDownloadTask.java create mode 100644 src/com/subgraph/orchid/directory/downloader/ConsensusFetcher.java delete mode 100644 src/com/subgraph/orchid/directory/downloader/DescriptorDownloadTask.java create mode 100644 src/com/subgraph/orchid/directory/downloader/DirectoryDocumentRequestor.java create mode 100644 src/com/subgraph/orchid/directory/downloader/DirectoryRequestFailedException.java create mode 100644 src/com/subgraph/orchid/directory/downloader/DocumentFetcher.java create mode 100644 src/com/subgraph/orchid/directory/downloader/MicrodescriptorFetcher.java create mode 100644 src/com/subgraph/orchid/directory/downloader/RouterDescriptorFetcher.java diff --git a/src/com/subgraph/orchid/DirectoryDownloader.java b/src/com/subgraph/orchid/DirectoryDownloader.java index 725a7119..8603cbb4 100644 --- a/src/com/subgraph/orchid/DirectoryDownloader.java +++ b/src/com/subgraph/orchid/DirectoryDownloader.java @@ -1,6 +1,26 @@ package com.subgraph.orchid; +import java.util.List; +import java.util.Set; + +import com.subgraph.orchid.ConsensusDocument.RequiredCertificate; +import com.subgraph.orchid.data.HexDigest; +import com.subgraph.orchid.directory.downloader.DirectoryRequestFailedException; + public interface DirectoryDownloader { void start(Directory directory); - RouterDescriptor downloadBridgeDescriptor(Router bridge); + + RouterDescriptor downloadBridgeDescriptor(Router bridge) throws DirectoryRequestFailedException; + + ConsensusDocument downloadCurrentConsensus(boolean useMicrodescriptors) throws DirectoryRequestFailedException; + ConsensusDocument downloadCurrentConsensus(boolean useMicrodescriptors, DirectoryCircuit circuit) throws DirectoryRequestFailedException; + + List downloadKeyCertificates(Set required) throws DirectoryRequestFailedException; + List downloadKeyCertificates(Set required, DirectoryCircuit circuit) throws DirectoryRequestFailedException; + + List downloadRouterDescriptors(Set fingerprints) throws DirectoryRequestFailedException; + List downloadRouterDescriptors(Set fingerprints, DirectoryCircuit circuit) throws DirectoryRequestFailedException; + + List downloadRouterMicrodescriptors(Set fingerprints) throws DirectoryRequestFailedException; + List downloadRouterMicrodescriptors(Set fingerprints, DirectoryCircuit circuit) throws DirectoryRequestFailedException; } diff --git a/src/com/subgraph/orchid/Tor.java b/src/com/subgraph/orchid/Tor.java index 57b04acb..3694192d 100644 --- a/src/com/subgraph/orchid/Tor.java +++ b/src/com/subgraph/orchid/Tor.java @@ -161,7 +161,7 @@ static public SocksPortListener createSocksPortListener(TorConfig config, Circui * @return A new DirectoryDownloader instance. * @see DirectoryDownloaderImpl */ - static public DirectoryDownloaderImpl createDirectoryDownloader(TorConfig config) { - return new DirectoryDownloaderImpl(config); + static public DirectoryDownloaderImpl createDirectoryDownloader(TorConfig config, TorInitializationTracker initializationTracker) { + return new DirectoryDownloaderImpl(config, initializationTracker); } } diff --git a/src/com/subgraph/orchid/TorClient.java b/src/com/subgraph/orchid/TorClient.java index 93161280..37301267 100644 --- a/src/com/subgraph/orchid/TorClient.java +++ b/src/com/subgraph/orchid/TorClient.java @@ -43,7 +43,7 @@ public TorClient() { initializationTracker = Tor.createInitalizationTracker(); initializationTracker.addListener(createReadyFlagInitializationListener()); connectionCache = Tor.createConnectionCache(config, initializationTracker); - directoryDownloader = Tor.createDirectoryDownloader(config); + directoryDownloader = Tor.createDirectoryDownloader(config, initializationTracker); circuitManager = Tor.createCircuitManager(config, directoryDownloader, directory, connectionCache, initializationTracker); socksListener = Tor.createSocksPortListener(config, circuitManager); readyLatch = new CountDownLatch(1); diff --git a/src/com/subgraph/orchid/circuits/guards/Bridges.java b/src/com/subgraph/orchid/circuits/guards/Bridges.java index 4385b140..d11cbe99 100644 --- a/src/com/subgraph/orchid/circuits/guards/Bridges.java +++ b/src/com/subgraph/orchid/circuits/guards/Bridges.java @@ -15,6 +15,7 @@ import com.subgraph.orchid.TorConfig; import com.subgraph.orchid.config.TorConfigBridgeLine; import com.subgraph.orchid.crypto.TorRandom; +import com.subgraph.orchid.directory.downloader.DirectoryRequestFailedException; public class Bridges { private static final Logger logger = Logger.getLogger(Bridges.class.getName()); @@ -37,14 +38,18 @@ public void run() { private void downloadDescriptor() { logger.fine("Downloading descriptor for bridge: "+ target); - final RouterDescriptor descriptor = directoryDownloader.downloadBridgeDescriptor(target); - if(descriptor != null) { - logger.fine("Descriptor received for bridge "+ target +". Adding to list of usable bridges"); - target.setDescriptor(descriptor); - synchronized(lock) { - bridgeRouters.add(target); - lock.notifyAll(); + try { + final RouterDescriptor descriptor = directoryDownloader.downloadBridgeDescriptor(target); + if(descriptor != null) { + logger.fine("Descriptor received for bridge "+ target +". Adding to list of usable bridges"); + target.setDescriptor(descriptor); + synchronized(lock) { + bridgeRouters.add(target); + lock.notifyAll(); + } } + } catch (DirectoryRequestFailedException e) { + logger.warning("Failed to download descriptor for bridge: "+ e.getMessage()); } } diff --git a/src/com/subgraph/orchid/circuits/hs/HSDescriptorDownloader.java b/src/com/subgraph/orchid/circuits/hs/HSDescriptorDownloader.java index fb557a2a..6a9d59ee 100644 --- a/src/com/subgraph/orchid/circuits/hs/HSDescriptorDownloader.java +++ b/src/com/subgraph/orchid/circuits/hs/HSDescriptorDownloader.java @@ -15,6 +15,7 @@ import com.subgraph.orchid.TorException; import com.subgraph.orchid.circuits.CircuitManagerImpl; import com.subgraph.orchid.directory.DocumentFieldParserImpl; +import com.subgraph.orchid.directory.downloader.DirectoryRequestFailedException; import com.subgraph.orchid.directory.downloader.HttpConnection; import com.subgraph.orchid.directory.parsing.DocumentParsingResultHandler; @@ -48,7 +49,6 @@ private HSDescriptor downloadDescriptorFrom(HSDescriptorDirectory dd) { Stream stream = null; try { - //stream = circuitManager.openDirectoryStreamTo(dd.getDirectory()); stream = openHSDirectoryStream(dd.getDirectory()); HttpConnection http = new HttpConnection(stream); http.sendGetRequest("/tor/rendezvous2/"+ dd.getDescriptorId().toBase32()); @@ -71,6 +71,9 @@ private HSDescriptor downloadDescriptorFrom(HSDescriptorDirectory dd) { } catch (OpenFailedException e) { logger.info("Failed to open stream to HS directory "+ dd.getDirectory() +" : "+ e.getMessage()); return null; + } catch (DirectoryRequestFailedException e) { + logger.info("Directory request to HS directory "+ dd.getDirectory() + " failed "+ e.getMessage()); + return null; } finally { if(stream != null) { stream.close(); @@ -88,7 +91,7 @@ private Stream openHSDirectoryStream(Router directory) throws TimeoutException, try { final DirectoryCircuit dc = circuit.cannibalizeToDirectory(directory); - return dc.openDirectoryStream(10000); + return dc.openDirectoryStream(10000, true); } catch (StreamConnectFailedException e) { circuit.markForClose(); throw new OpenFailedException("Failed to open directory stream"); diff --git a/src/com/subgraph/orchid/directory/downloader/AbstractDirectoryDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/AbstractDirectoryDownloadTask.java deleted file mode 100644 index b534fb78..00000000 --- a/src/com/subgraph/orchid/directory/downloader/AbstractDirectoryDownloadTask.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.subgraph.orchid.directory.downloader; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; - -import com.subgraph.orchid.Directory; -import com.subgraph.orchid.OpenFailedException; -import com.subgraph.orchid.Stream; -import com.subgraph.orchid.TorException; -import com.subgraph.orchid.data.HexDigest; -import com.subgraph.orchid.directory.parsing.DocumentParserFactory; - - -public abstract class AbstractDirectoryDownloadTask implements Runnable { - protected final static Logger logger = Logger.getLogger(AbstractDirectoryDownloadTask.class.getName()); - private final static boolean USE_COMPRESSION = true; - - private final DirectoryDownloadTask downloader; - private final int purposeCode; - - protected AbstractDirectoryDownloadTask(DirectoryDownloadTask downloader, int purposeCode) { - this.downloader = downloader; - this.purposeCode = purposeCode; - } - - protected Directory getDirectory() { - return downloader.getDirectory(); - } - - protected DocumentParserFactory getParserFactory() { - return downloader.getDocumentParserFactory(); - } - - protected HttpConnection openDirectoryConnection() throws InterruptedException, TimeoutException, OpenFailedException { - final Stream stream = downloader.getCircuitManager().openDirectoryStream(purposeCode); - return new HttpConnection(stream); - } - - public void run() { - try { - makeRequest(); - } catch(TorException e) { - logger.info(e.getMessage()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (TimeoutException e) { - logger.warning("Timeout opening directory stream"); - } catch (OpenFailedException e) { - logger.warning("Failed to open directory stream"); - } finally { - finishRequest(downloader); - } - } - - private void makeRequest() throws InterruptedException, TimeoutException, OpenFailedException { - final HttpConnection http = openDirectoryConnection(); - final String request = getRequestPath(); - ByteBuffer response = null; - try { - logger.fine("request to "+ http.getHost() + " : "+ request); - response = requestDocument(http, request); - processResponse(response, http); - } catch(IOException e) { - logger.warning("IO error making request "+ request +" to host ["+ http.getHost() + "]: "+ e); - } finally { - http.close(); - } - } - - abstract protected String getRequestPath(); - abstract protected void processResponse(ByteBuffer response, HttpConnection http); - abstract protected void finishRequest(DirectoryDownloadTask downloader); - - protected ByteBuffer requestDocument(HttpConnection connection, String request) throws IOException { - if(USE_COMPRESSION) { - request += ".z"; - } - connection.sendGetRequest(request); - connection.readResponse(); - if(connection.getStatusCode() == 200) { - return connection.getMessageBody(); - } - throw new TorException("Request "+ request +" to directory "+ - connection.getHost() +" returned error code: "+ - connection.getStatusCode() + " "+ connection.getStatusMessage()); - } - - protected String fingerprintsToRequestString(List fingerprints, boolean useMicrodescriptors) { - final StringBuilder sb = new StringBuilder(); - for(HexDigest fp: fingerprints) { - if(useMicrodescriptors) { - appendMicrodescriptor(sb, fp); - } else { - appendFingerprint(sb, fp); - } - } - return sb.toString(); - } - - private void appendFingerprint(StringBuilder sb, HexDigest fp) { - if(sb.length() > 0) { - sb.append("+"); - } - sb.append(fp.toString()); - } - - private void appendMicrodescriptor(StringBuilder sb, HexDigest md) { - if(sb.length() > 0) { - sb.append("-"); - } - sb.append(md.toBase64(true)); - } -} diff --git a/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorDownloadTask.java deleted file mode 100644 index 04769d97..00000000 --- a/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorDownloadTask.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.subgraph.orchid.directory.downloader; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; - -import com.subgraph.orchid.CircuitManager; -import com.subgraph.orchid.DirectoryCircuit; -import com.subgraph.orchid.OpenFailedException; -import com.subgraph.orchid.Router; -import com.subgraph.orchid.RouterDescriptor; -import com.subgraph.orchid.Stream; -import com.subgraph.orchid.StreamConnectFailedException; -import com.subgraph.orchid.TorException; -import com.subgraph.orchid.directory.parsing.BasicDocumentParsingResult; -import com.subgraph.orchid.directory.parsing.DocumentParser; -import com.subgraph.orchid.directory.parsing.DocumentParserFactory; - -public class BridgeDescriptorDownloadTask implements Callable { - private final static Logger logger = Logger.getLogger(BridgeDescriptorDownloadTask.class.getName()); - - private final DocumentParserFactory parserFactory; - private final CircuitManager circuitManager; - private final Router target; - - public BridgeDescriptorDownloadTask(DocumentParserFactory parserFactory, CircuitManager circuitManager, Router target) { - this.parserFactory = parserFactory; - this.circuitManager = circuitManager; - this.target = target; - } - - public RouterDescriptor call() throws OpenFailedException, IOException, InterruptedException, TimeoutException, StreamConnectFailedException, TorException { - final HttpConnection connection = openConnection(); - try { - final ByteBuffer body = requestDocument(connection); - return processResponse(body); - } finally { - connection.close(); - } - } - - private HttpConnection openConnection() throws OpenFailedException, InterruptedException, TimeoutException, StreamConnectFailedException { - DirectoryCircuit circuit = circuitManager.openDirectoryCircuitTo(Arrays.asList(target)); - Stream stream = circuit.openDirectoryStream(2000); - return new HttpConnection(stream); - } - - private ByteBuffer requestDocument(HttpConnection connection) throws IOException { - connection.sendGetRequest("/tor/server/authority"); - connection.readResponse(); - if(connection.getStatusCode() == 200) { - return connection.getMessageBody(); - } - throw new TorException("Request /tor/server/authority to bridge "+ target - +" returned error code: "+ connection.getStatusCode() +" "+ connection.getStatusMessage()); - } - - private RouterDescriptor processResponse(ByteBuffer body) { - final DocumentParser parser = parserFactory.createRouterDescriptorParser(body, true); - final BasicDocumentParsingResult result = new BasicDocumentParsingResult(); - if(parser.parse(result) && !result.isError()) { - logger.fine("Valid descriptor received from bridge "+ target); - return result.getDocument(); - } - logger.warning("Failed to parse descriptor returned from bridge: "+ target +" ("+ result.getMessage() + ")"); - return null; - } - - - - - - - -} diff --git a/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorFetcher.java b/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorFetcher.java new file mode 100644 index 00000000..4514bbd6 --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/BridgeDescriptorFetcher.java @@ -0,0 +1,19 @@ +package com.subgraph.orchid.directory.downloader; + +import java.nio.ByteBuffer; + +import com.subgraph.orchid.RouterDescriptor; +import com.subgraph.orchid.directory.parsing.DocumentParser; + +public class BridgeDescriptorFetcher extends DocumentFetcher{ + + @Override + String getRequestPath() { + return "/tor/server/authority"; + } + + @Override + DocumentParser createParser(ByteBuffer response) { + return PARSER_FACTORY.createRouterDescriptorParser(response, true); + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/CertificateDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/CertificateDownloadTask.java deleted file mode 100644 index 84bab189..00000000 --- a/src/com/subgraph/orchid/directory/downloader/CertificateDownloadTask.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.subgraph.orchid.directory.downloader; - -import java.nio.ByteBuffer; -import java.util.Set; - -import com.subgraph.orchid.CircuitManager; -import com.subgraph.orchid.ConsensusDocument.RequiredCertificate; -import com.subgraph.orchid.KeyCertificate; -import com.subgraph.orchid.directory.parsing.DocumentParser; -import com.subgraph.orchid.directory.parsing.DocumentParsingResultHandler; - -public class CertificateDownloadTask extends AbstractDirectoryDownloadTask{ - - private final Set requiredCertificates; - - CertificateDownloadTask(Set requiredCertificates, DirectoryDownloadTask downloader) { - super(downloader, CircuitManager.DIRECTORY_PURPOSE_CERTIFICATES); - this.requiredCertificates = requiredCertificates; - } - - @Override - protected String getRequestPath() { - return "/tor/keys/fp-sk/"+ getRequiredCertificatesRequestString(); - } - - private String getRequiredCertificatesRequestString() { - final StringBuilder sb = new StringBuilder(); - for(RequiredCertificate rc: requiredCertificates) { - if(sb.length() > 0) { - sb.append("+"); - } - sb.append(rc.getAuthorityIdentity().toString()); - sb.append("-"); - sb.append(rc.getSigningKey().toString()); - } - return sb.toString(); - } - - @Override - protected void processResponse(ByteBuffer response, final HttpConnection http) { - final DocumentParser parser = getParserFactory().createKeyCertificateParser(response); - final boolean success = parser.parse(new DocumentParsingResultHandler() { - - public void parsingError(String message) { - logger.warning("Parsing error processing certificate document from ["+ http.getHost() +"]: "+ message); - } - - public void documentParsed(KeyCertificate document) { - getDirectory().addCertificate(document); - } - - public void documentInvalid(KeyCertificate document, String message) { - logger.warning("Received invalid certificate document: " + message); - } - }); - - if(success) { - getDirectory().storeCertificates(); - } - } - - @Override - protected void finishRequest(DirectoryDownloadTask downloader) { - downloader.clearDownloadingCertificates(); - } -} diff --git a/src/com/subgraph/orchid/directory/downloader/CertificateFetcher.java b/src/com/subgraph/orchid/directory/downloader/CertificateFetcher.java new file mode 100644 index 00000000..799d8e9a --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/CertificateFetcher.java @@ -0,0 +1,40 @@ +package com.subgraph.orchid.directory.downloader; + +import java.nio.ByteBuffer; +import java.util.Set; + +import com.subgraph.orchid.KeyCertificate; +import com.subgraph.orchid.ConsensusDocument.RequiredCertificate; +import com.subgraph.orchid.directory.parsing.DocumentParser; + +public class CertificateFetcher extends DocumentFetcher{ + + private final Set requiredCertificates; + + public CertificateFetcher(Set requiredCertificates) { + this.requiredCertificates = requiredCertificates; + } + + @Override + String getRequestPath() { + return "/tor/keys/fp-sk/"+ getRequiredCertificatesRequestString(); + } + + private String getRequiredCertificatesRequestString() { + final StringBuilder sb = new StringBuilder(); + for(RequiredCertificate rc: requiredCertificates) { + if(sb.length() > 0) { + sb.append("+"); + } + sb.append(rc.getAuthorityIdentity().toString()); + sb.append("-"); + sb.append(rc.getSigningKey().toString()); + } + return sb.toString(); + } + + @Override + DocumentParser createParser(ByteBuffer response) { + return PARSER_FACTORY.createKeyCertificateParser(response); + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/ConsensusDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/ConsensusDownloadTask.java deleted file mode 100644 index 8f26247e..00000000 --- a/src/com/subgraph/orchid/directory/downloader/ConsensusDownloadTask.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.subgraph.orchid.directory.downloader; - -import java.nio.ByteBuffer; - -import com.subgraph.orchid.CircuitManager; -import com.subgraph.orchid.ConsensusDocument; -import com.subgraph.orchid.directory.parsing.DocumentParser; -import com.subgraph.orchid.directory.parsing.DocumentParsingResultHandler; - -public class ConsensusDownloadTask extends AbstractDirectoryDownloadTask { - - - private ConsensusDocument newConsensusDocument = null; - private final boolean useMicrodescriptors; - - ConsensusDownloadTask(DirectoryDownloadTask downloader, boolean useMicrodescriptors) { - super(downloader, CircuitManager.DIRECTORY_PURPOSE_CONSENSUS); - this.useMicrodescriptors = useMicrodescriptors; - } - - @Override - protected String getRequestPath() { - if(useMicrodescriptors) { - return "/tor/status-vote/current/consensus-microdesc"; - } else { - return "/tor/status-vote/current/consensus"; - } - } - - @Override - protected void processResponse(ByteBuffer response, final HttpConnection http) { - final DocumentParser parser = getParserFactory().createConsensusDocumentParser(response); - final boolean success = parser.parse(new DocumentParsingResultHandler() { - - public void parsingError(String message) { - logger.warning("Parsing error processing consensus document from ["+ http.getHost() +"]: "+ message); - } - - public void documentParsed(ConsensusDocument document) { - newConsensusDocument = document; - getDirectory().addConsensusDocument(document, false); - } - - public void documentInvalid(ConsensusDocument document, String message) { - logger.warning("Received consensus document is invalid: "+ message); - } - }); - - if(success) { - getDirectory().storeConsensus(); - } - } - - @Override - protected void finishRequest(DirectoryDownloadTask downloader) { - if(newConsensusDocument != null) { - downloader.setCurrentConsensus(newConsensusDocument); - } - downloader.clearDownloadingConsensus(); - } -} diff --git a/src/com/subgraph/orchid/directory/downloader/ConsensusFetcher.java b/src/com/subgraph/orchid/directory/downloader/ConsensusFetcher.java new file mode 100644 index 00000000..9e92e09f --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/ConsensusFetcher.java @@ -0,0 +1,29 @@ +package com.subgraph.orchid.directory.downloader; + +import java.nio.ByteBuffer; + +import com.subgraph.orchid.ConsensusDocument; +import com.subgraph.orchid.directory.parsing.DocumentParser; + +public class ConsensusFetcher extends DocumentFetcher{ + + private final static String CONSENSUS_BASE_PATH = "/tor/status-vote/current/"; + + private final boolean useMicrodescriptors; + + + public ConsensusFetcher(boolean useMicrodescriptors) { + this.useMicrodescriptors = useMicrodescriptors; + } + + @Override + String getRequestPath() { + return CONSENSUS_BASE_PATH + ((useMicrodescriptors) ? + ("consensus-microdesc") : ("consensus")); + } + + @Override + DocumentParser createParser(ByteBuffer response) { + return PARSER_FACTORY.createConsensusDocumentParser(response); + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/DescriptorDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/DescriptorDownloadTask.java deleted file mode 100644 index ca17511b..00000000 --- a/src/com/subgraph/orchid/directory/downloader/DescriptorDownloadTask.java +++ /dev/null @@ -1,110 +0,0 @@ -package com.subgraph.orchid.directory.downloader; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.subgraph.orchid.CircuitManager; -import com.subgraph.orchid.RouterDescriptor; -import com.subgraph.orchid.RouterMicrodescriptor; -import com.subgraph.orchid.data.HexDigest; -import com.subgraph.orchid.directory.parsing.DocumentParser; -import com.subgraph.orchid.directory.parsing.DocumentParsingResultHandler; - -public class DescriptorDownloadTask extends AbstractDirectoryDownloadTask{ - - private final List fingerprints; - private final boolean useMicrodescriptors; - - DescriptorDownloadTask(List fingerprints, DirectoryDownloadTask downloader, boolean useMicrodescriptors) { - super(downloader, CircuitManager.DIRECTORY_PURPOSE_DESCRIPTORS); - this.fingerprints = fingerprints; - this.useMicrodescriptors = useMicrodescriptors; - } - - @Override - protected String getRequestPath() { - final String fps = fingerprintsToRequestString(fingerprints, useMicrodescriptors); - if(useMicrodescriptors) { - return "/tor/micro/d/" + fps; - } else { - return "/tor/server/d/" + fps; - } - } - - @Override - protected void processResponse(ByteBuffer response, final HttpConnection http) { - if(useMicrodescriptors) { - processMicrodescriptorResponse(response, http); - } else { - processDescriptorResponse(response, http); - } - } - - private void processDescriptorResponse(ByteBuffer response, final HttpConnection http) { - final Set requested = new HashSet(); - requested.addAll(fingerprints); - - final DocumentParser parser = getParserFactory().createRouterDescriptorParser(response, true); - final List descriptors = new ArrayList(); - final boolean success = parser.parse(new DocumentParsingResultHandler() { - - public void parsingError(String message) { - logger.warning("Parsing error processing router descriptors from ["+ http.getHost() +"]: "+ message); - } - - public void documentParsed(RouterDescriptor document) { - if(!requested.contains(document.getDescriptorDigest())) { - logger.warning("Server returned a router descriptor that was not requested. Ignoring."); - return; - } - descriptors.add(document); - } - - public void documentInvalid(RouterDescriptor document, String message) { - logger.warning("Router descriptor "+ document.getNickname() +" invalid: "+ message); - getDirectory().markDescriptorInvalid(document); - } - }); - - if(success) { - getDirectory().addRouterDescriptors(descriptors); - } - } - - private void processMicrodescriptorResponse(ByteBuffer response, final HttpConnection http) { - final Set requested = new HashSet(); - requested.addAll(fingerprints); - final DocumentParser parser = getParserFactory().createRouterMicrodescriptorParser(response); - final List microdescriptors = new ArrayList(); - boolean success = parser.parse(new DocumentParsingResultHandler() { - - public void parsingError(String message) { - logger.warning("Parsing error processing microdescriptor from ["+ http.getHost() + "]: "+ message); - } - - public void documentParsed(RouterMicrodescriptor document) { - if(!requested.contains(document.getDescriptorDigest())) { - logger.warning("Server returned a router microdescriptor that was not requested. Ignoring."); - return; - } - microdescriptors.add(document); - } - - public void documentInvalid(RouterMicrodescriptor document, String message) { - logger.warning("Invalid router microdescriptor returned: "+ message); - } - }); - - if(success) { - getDirectory().addRouterMicrodescriptors(microdescriptors); - } - } - - @Override - protected void finishRequest(DirectoryDownloadTask downloader) { - downloader.clearDownloadingDescriptors(); - } -} diff --git a/src/com/subgraph/orchid/directory/downloader/DirectoryDocumentRequestor.java b/src/com/subgraph/orchid/directory/downloader/DirectoryDocumentRequestor.java new file mode 100644 index 00000000..347c38c5 --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/DirectoryDocumentRequestor.java @@ -0,0 +1,125 @@ +package com.subgraph.orchid.directory.downloader; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeoutException; + +import com.subgraph.orchid.CircuitManager; +import com.subgraph.orchid.ConsensusDocument; +import com.subgraph.orchid.ConsensusDocument.RequiredCertificate; +import com.subgraph.orchid.DirectoryCircuit; +import com.subgraph.orchid.KeyCertificate; +import com.subgraph.orchid.Router; +import com.subgraph.orchid.RouterDescriptor; +import com.subgraph.orchid.RouterMicrodescriptor; +import com.subgraph.orchid.Stream; +import com.subgraph.orchid.StreamConnectFailedException; +import com.subgraph.orchid.Tor; +import com.subgraph.orchid.circuits.TorInitializationTracker; +import com.subgraph.orchid.data.HexDigest; + +/** + * Synchronously downloads directory documents. + */ +public class DirectoryDocumentRequestor { + private final static int OPEN_DIRECTORY_STREAM_TIMEOUT = 10 * 1000; + + private final DirectoryCircuit circuit; + private final TorInitializationTracker initializationTracker; + + + public DirectoryDocumentRequestor(DirectoryCircuit circuit) { + this(circuit, null); + } + + public DirectoryDocumentRequestor(DirectoryCircuit circuit, TorInitializationTracker initializationTracker) { + this.circuit = circuit; + this.initializationTracker = initializationTracker; + } + + public RouterDescriptor downloadBridgeDescriptor(Router bridge) throws DirectoryRequestFailedException { + return fetchSingleDocument(new BridgeDescriptorFetcher()); + } + + public ConsensusDocument downloadCurrentConsensus(boolean useMicrodescriptors) throws DirectoryRequestFailedException { + return fetchSingleDocument(new ConsensusFetcher(useMicrodescriptors), CircuitManager.DIRECTORY_PURPOSE_CONSENSUS); + } + + public List downloadKeyCertificates(Set required) throws DirectoryRequestFailedException { + return fetchDocuments(new CertificateFetcher(required), CircuitManager.DIRECTORY_PURPOSE_CERTIFICATES); + } + + public List downloadRouterDescriptors(Set fingerprints) throws DirectoryRequestFailedException { + return fetchDocuments(new RouterDescriptorFetcher(fingerprints), CircuitManager.DIRECTORY_PURPOSE_DESCRIPTORS); + } + + public List downloadRouterMicrodescriptors(Set fingerprints) throws DirectoryRequestFailedException { + return fetchDocuments(new MicrodescriptorFetcher(fingerprints), CircuitManager.DIRECTORY_PURPOSE_DESCRIPTORS); + } + + private T fetchSingleDocument(DocumentFetcher fetcher) throws DirectoryRequestFailedException { + return fetchSingleDocument(fetcher, 0); + } + + private T fetchSingleDocument(DocumentFetcher fetcher, int purpose) throws DirectoryRequestFailedException { + final List result = fetchDocuments(fetcher, purpose); + if(result.size() == 1) { + return result.get(0); + } + return null; + } + + private List fetchDocuments(DocumentFetcher fetcher, int purpose) throws DirectoryRequestFailedException { + try { + final HttpConnection http = createHttpConnection(purpose); + try { + return fetcher.requestDocuments(http); + } finally { + http.close(); + } + } catch (TimeoutException e) { + throw new DirectoryRequestFailedException("Directory request timed out"); + } catch (StreamConnectFailedException e) { + throw new DirectoryRequestFailedException("Failed to open directory stream", e); + } catch (IOException e) { + throw new DirectoryRequestFailedException("I/O exception processing directory request", e); + } catch (InterruptedException e) { + throw new DirectoryRequestFailedException("Directory request interrupted"); + } + } + + private HttpConnection createHttpConnection(int purpose) throws InterruptedException, TimeoutException, StreamConnectFailedException { + return new HttpConnection(openDirectoryStream(purpose)); + } + + private Stream openDirectoryStream(int purpose) throws InterruptedException, TimeoutException, StreamConnectFailedException { + final int requestEventCode = purposeToEventCode(purpose, false); + final int loadingEventCode = purposeToEventCode(purpose, true); + + notifyInitialization(requestEventCode); + + final Stream stream = circuit.openDirectoryStream(OPEN_DIRECTORY_STREAM_TIMEOUT, true); + notifyInitialization(loadingEventCode); + return stream; + } + + private int purposeToEventCode(int purpose, boolean getLoadingEvent) { + switch(purpose) { + case CircuitManager.DIRECTORY_PURPOSE_CONSENSUS: + return getLoadingEvent ? Tor.BOOTSTRAP_STATUS_LOADING_STATUS : Tor.BOOTSTRAP_STATUS_REQUESTING_STATUS; + case CircuitManager.DIRECTORY_PURPOSE_CERTIFICATES: + return getLoadingEvent ? Tor.BOOTSTRAP_STATUS_LOADING_KEYS : Tor.BOOTSTRAP_STATUS_REQUESTING_KEYS; + case CircuitManager.DIRECTORY_PURPOSE_DESCRIPTORS: + return getLoadingEvent ? Tor.BOOTSTRAP_STATUS_LOADING_DESCRIPTORS : Tor.BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS; + default: + return 0; + } + } + + private void notifyInitialization(int code) { + if(code > 0 && initializationTracker != null) { + initializationTracker.notifyEvent(code); + } + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/DirectoryDownloadTask.java b/src/com/subgraph/orchid/directory/downloader/DirectoryDownloadTask.java index 06b59c41..d0f51380 100644 --- a/src/com/subgraph/orchid/directory/downloader/DirectoryDownloadTask.java +++ b/src/com/subgraph/orchid/directory/downloader/DirectoryDownloadTask.java @@ -1,22 +1,24 @@ package com.subgraph.orchid.directory.downloader; +import java.util.Collection; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; -import com.subgraph.orchid.CircuitManager; import com.subgraph.orchid.ConsensusDocument; import com.subgraph.orchid.Directory; +import com.subgraph.orchid.DirectoryDownloader; +import com.subgraph.orchid.KeyCertificate; import com.subgraph.orchid.TorConfig; import com.subgraph.orchid.TorConfig.AutoBoolValue; import com.subgraph.orchid.crypto.TorRandom; import com.subgraph.orchid.data.HexDigest; import com.subgraph.orchid.data.Timestamp; -import com.subgraph.orchid.directory.DocumentParserFactoryImpl; -import com.subgraph.orchid.directory.parsing.DocumentParserFactory; public class DirectoryDownloadTask implements Runnable { private final static Logger logger = Logger.getLogger(DirectoryDownloadTask.class.getName()); @@ -24,8 +26,7 @@ public class DirectoryDownloadTask implements Runnable { private final TorConfig config; private final Directory directory; - private final CircuitManager circuitManager; - private final DocumentParserFactory parserFactory; + private final DirectoryDownloader downloader; private final TorRandom random; private final DescriptorProcessor descriptorProcessor; @@ -39,11 +40,10 @@ public class DirectoryDownloadTask implements Runnable { private ConsensusDocument currentConsensus; private Date consensusDownloadTime; - DirectoryDownloadTask(TorConfig config, Directory directory, CircuitManager circuitManager) { + DirectoryDownloadTask(TorConfig config, Directory directory, DirectoryDownloader downloader) { this.config = config; this.directory = directory; - this.circuitManager = circuitManager; - this.parserFactory = new DocumentParserFactoryImpl(); + this.downloader = downloader; this.random = new TorRandom(); this.outstandingDescriptorTasks = new AtomicInteger(); this.descriptorProcessor = new DescriptorProcessor(config, directory); @@ -67,39 +67,14 @@ public void run() { } } - Directory getDirectory() { - return directory; - } - - CircuitManager getCircuitManager() { - return circuitManager; - } - - DocumentParserFactory getDocumentParserFactory() { - return parserFactory; - } - - void clearDownloadingCertificates() { - isDownloadingCertificates = false; - } - - void clearDownloadingConsensus() { - isDownloadingConsensus = false; - } - - void clearDownloadingDescriptors() { - outstandingDescriptorTasks.decrementAndGet(); - } - private void checkCertificates() { if (isDownloadingCertificates || directory.getRequiredCertificates().isEmpty()) { return; } - CertificateDownloadTask task = new CertificateDownloadTask(directory.getRequiredCertificates(), this); isDownloadingCertificates = true; - executor.execute(task); + executor.execute(new DownloadCertificatesTask()); } void setCurrentConsensus(ConsensusDocument consensus) { @@ -164,9 +139,9 @@ private void checkConsensus() { if (isDownloadingConsensus || !needConsensusDownload()) { return; } - ConsensusDownloadTask task = new ConsensusDownloadTask(this, useMicrodescriptors()); + isDownloadingConsensus = true; - executor.execute(task); + executor.execute(new DownloadConsensusTask()); } private void checkDescriptors() { @@ -179,9 +154,8 @@ private void checkDescriptors() { return; } for (List dlist : ds) { - DescriptorDownloadTask task = new DescriptorDownloadTask(dlist, this, useMicrodescriptors()); outstandingDescriptorTasks.incrementAndGet(); - executor.execute(task); + executor.execute(new DownloadRouterDescriptorsTask(dlist, useMicrodescriptors())); } } @@ -189,6 +163,57 @@ private boolean useMicrodescriptors() { return config.getUseMicrodescriptors() != AutoBoolValue.FALSE; } + private class DownloadConsensusTask implements Runnable { + public void run() { + try { + final ConsensusDocument consensus = downloader.downloadCurrentConsensus(useMicrodescriptors()); + setCurrentConsensus(consensus); + directory.addConsensusDocument(consensus, false); + + } catch (DirectoryRequestFailedException e) { + logger.warning("Failed to download current consensus document: "+ e.getMessage()); + } finally { + isDownloadingConsensus = false; + } + } + } + private class DownloadRouterDescriptorsTask implements Runnable { + private final Set fingerprints; + private final boolean useMicrodescriptors; + + public DownloadRouterDescriptorsTask(Collection fingerprints, boolean useMicrodescriptors) { + this.fingerprints = new HashSet(fingerprints); + this.useMicrodescriptors = useMicrodescriptors; + } + + public void run() { + try { + if(useMicrodescriptors) { + directory.addRouterMicrodescriptors(downloader.downloadRouterMicrodescriptors(fingerprints)); + } else { + directory.addRouterDescriptors(downloader.downloadRouterDescriptors(fingerprints)); + } + } catch (DirectoryRequestFailedException e) { + logger.warning("Failed to download router descriptors: "+ e.getMessage()); + } finally { + outstandingDescriptorTasks.decrementAndGet(); + } + } + } + private class DownloadCertificatesTask implements Runnable { + public void run() { + try { + for(KeyCertificate c: downloader.downloadKeyCertificates(directory.getRequiredCertificates())) { + directory.addCertificate(c); + } + directory.storeCertificates(); + } catch (DirectoryRequestFailedException e) { + logger.warning("Failed to download key certificates: "+ e.getMessage()); + } finally { + isDownloadingCertificates = false; + } + } + } } diff --git a/src/com/subgraph/orchid/directory/downloader/DirectoryDownloaderImpl.java b/src/com/subgraph/orchid/directory/downloader/DirectoryDownloaderImpl.java index cd8a7bca..e5852a40 100644 --- a/src/com/subgraph/orchid/directory/downloader/DirectoryDownloaderImpl.java +++ b/src/com/subgraph/orchid/directory/downloader/DirectoryDownloaderImpl.java @@ -1,34 +1,40 @@ package com.subgraph.orchid.directory.downloader; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; import com.subgraph.orchid.CircuitManager; +import com.subgraph.orchid.ConsensusDocument; +import com.subgraph.orchid.ConsensusDocument.RequiredCertificate; +import com.subgraph.orchid.Descriptor; import com.subgraph.orchid.Directory; +import com.subgraph.orchid.DirectoryCircuit; import com.subgraph.orchid.DirectoryDownloader; +import com.subgraph.orchid.KeyCertificate; +import com.subgraph.orchid.OpenFailedException; import com.subgraph.orchid.Router; import com.subgraph.orchid.RouterDescriptor; +import com.subgraph.orchid.RouterMicrodescriptor; import com.subgraph.orchid.TorConfig; -import com.subgraph.orchid.directory.DocumentParserFactoryImpl; -import com.subgraph.orchid.directory.parsing.DocumentParserFactory; +import com.subgraph.orchid.circuits.TorInitializationTracker; +import com.subgraph.orchid.data.HexDigest; public class DirectoryDownloaderImpl implements DirectoryDownloader { private final static Logger logger = Logger.getLogger(DirectoryDownloaderImpl.class.getName()); - private final static DocumentParserFactory parserFactory = new DocumentParserFactoryImpl(); private final TorConfig config; - private final ExecutorService executor; + private final TorInitializationTracker initializationTracker; private CircuitManager circuitManager; private boolean isStarted; private Thread downloadTaskThread; - public DirectoryDownloaderImpl(TorConfig config) { + public DirectoryDownloaderImpl(TorConfig config, TorInitializationTracker initializationTracker) { this.config = config; - this.executor = Executors.newCachedThreadPool(); + this.initializationTracker = initializationTracker; } public void setCircuitManager(CircuitManager circuitManager) { @@ -44,27 +50,85 @@ public synchronized void start(Directory directory) { throw new IllegalStateException("Must set CircuitManager instance with setCircuitManager() before starting."); } - final DirectoryDownloadTask task = new DirectoryDownloadTask(config, directory, circuitManager); + final DirectoryDownloadTask task = new DirectoryDownloadTask(config, directory, this); downloadTaskThread = new Thread(task); downloadTaskThread.start(); isStarted = true; } - public RouterDescriptor downloadBridgeDescriptor(Router bridge) { - if(circuitManager == null) { - throw new IllegalStateException("Must set CircuitManager instance with setCircuitManager()"); + public RouterDescriptor downloadBridgeDescriptor(Router bridge) throws DirectoryRequestFailedException { + final DirectoryDocumentRequestor requestor = new DirectoryDocumentRequestor(openBridgeCircuit(bridge)); + return requestor.downloadBridgeDescriptor(bridge); + } + + + public ConsensusDocument downloadCurrentConsensus(boolean useMicrodescriptors) throws DirectoryRequestFailedException { + return downloadCurrentConsensus(useMicrodescriptors, openCircuit()); + } + + public ConsensusDocument downloadCurrentConsensus(boolean useMicrodescriptors, DirectoryCircuit circuit) throws DirectoryRequestFailedException { + final DirectoryDocumentRequestor requestor = new DirectoryDocumentRequestor(circuit, initializationTracker); + return requestor.downloadCurrentConsensus(useMicrodescriptors); + } + + public List downloadKeyCertificates(Set required) throws DirectoryRequestFailedException { + return downloadKeyCertificates(required, openCircuit()); + } + + public List downloadKeyCertificates(Set required, DirectoryCircuit circuit) throws DirectoryRequestFailedException { + final DirectoryDocumentRequestor requestor = new DirectoryDocumentRequestor(circuit, initializationTracker); + return requestor.downloadKeyCertificates(required); + } + + public List downloadRouterDescriptors(Set fingerprints) throws DirectoryRequestFailedException { + return downloadRouterDescriptors(fingerprints, openCircuit()); + } + + public List downloadRouterDescriptors(Set fingerprints, DirectoryCircuit circuit) throws DirectoryRequestFailedException { + final DirectoryDocumentRequestor requestor = new DirectoryDocumentRequestor(circuit, initializationTracker); + final List ds = requestor.downloadRouterDescriptors(fingerprints); + return removeUnrequestedDescriptors(fingerprints, ds); + } + + public List downloadRouterMicrodescriptors(Set fingerprints) throws DirectoryRequestFailedException { + return downloadRouterMicrodescriptors(fingerprints, openCircuit()); + } + + public List downloadRouterMicrodescriptors(Set fingerprints, DirectoryCircuit circuit) throws DirectoryRequestFailedException { + final DirectoryDocumentRequestor requestor = new DirectoryDocumentRequestor(circuit, initializationTracker); + final List ds = requestor.downloadRouterMicrodescriptors(fingerprints); + return removeUnrequestedDescriptors(fingerprints, ds); + } + + private List removeUnrequestedDescriptors(Set requested, List received) { + final List result = new ArrayList(); + int unrequestedCount = 0; + for(T d: received) { + if(requested.contains(d.getDescriptorDigest())) { + result.add(d); + } else { + unrequestedCount += 1; + } + } + if(unrequestedCount > 0) { + logger.warning("Discarding "+ unrequestedCount + " received descriptor(s) with fingerprints that did not match requested descriptors"); + } + return result; + } + + private DirectoryCircuit openCircuit() throws DirectoryRequestFailedException { + try { + return circuitManager.openDirectoryCircuit(); + } catch (OpenFailedException e) { + throw new DirectoryRequestFailedException("Failed to open directory circuit", e); } - BridgeDescriptorDownloadTask task = new BridgeDescriptorDownloadTask(parserFactory, circuitManager, bridge); - Future future = executor.submit(task); + } + + private DirectoryCircuit openBridgeCircuit(Router bridge) throws DirectoryRequestFailedException { try { - return future.get(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - logger.warning("Failed to download bridge descriptor for "+ bridge +" : "+ e.getCause().getMessage()); + return circuitManager.openDirectoryCircuitTo(Arrays.asList(bridge)); + } catch (OpenFailedException e) { + throw new DirectoryRequestFailedException("Failed to open directory circuit to bridge "+ bridge, e); } - return null; } - - } diff --git a/src/com/subgraph/orchid/directory/downloader/DirectoryRequestFailedException.java b/src/com/subgraph/orchid/directory/downloader/DirectoryRequestFailedException.java new file mode 100644 index 00000000..c41df470 --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/DirectoryRequestFailedException.java @@ -0,0 +1,15 @@ +package com.subgraph.orchid.directory.downloader; + +public class DirectoryRequestFailedException extends Exception { + + private static final long serialVersionUID = 1L; + + public DirectoryRequestFailedException(String message) { + super(message); + } + + public DirectoryRequestFailedException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/com/subgraph/orchid/directory/downloader/DocumentFetcher.java b/src/com/subgraph/orchid/directory/downloader/DocumentFetcher.java new file mode 100644 index 00000000..5cf73baa --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/DocumentFetcher.java @@ -0,0 +1,52 @@ +package com.subgraph.orchid.directory.downloader; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +import com.subgraph.orchid.directory.DocumentParserFactoryImpl; +import com.subgraph.orchid.directory.parsing.BasicDocumentParsingResult; +import com.subgraph.orchid.directory.parsing.DocumentParser; +import com.subgraph.orchid.directory.parsing.DocumentParserFactory; + +public abstract class DocumentFetcher { + protected final static DocumentParserFactory PARSER_FACTORY = new DocumentParserFactoryImpl(); + + + abstract String getRequestPath(); + abstract DocumentParser createParser(ByteBuffer response); + + public List requestDocuments(HttpConnection httpConnection) throws IOException, DirectoryRequestFailedException { + final ByteBuffer body = makeRequest(httpConnection); + if(body.hasRemaining()) { + return processResponse(body); + }else { + return Collections.emptyList(); + } + } + + private ByteBuffer makeRequest(HttpConnection httpConnection) throws IOException, DirectoryRequestFailedException { + + httpConnection.sendGetRequest(getRequestPath()); + httpConnection.readResponse(); + if(httpConnection.getStatusCode() == 200) { + return httpConnection.getMessageBody(); + } + + throw new DirectoryRequestFailedException("Request "+ getRequestPath() +" to directory "+ + httpConnection.getHost() +" returned error code: "+ + httpConnection.getStatusCode() + " "+ httpConnection.getStatusMessage()); + + } + + private List processResponse(ByteBuffer response) throws DirectoryRequestFailedException { + final DocumentParser parser = createParser(response); + final BasicDocumentParsingResult result = new BasicDocumentParsingResult(); + final boolean success = parser.parse(result); + if(success) { + return result.getParsedDocuments(); + } + throw new DirectoryRequestFailedException("Failed to parse response from directory: "+ result.getMessage()); + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/HttpConnection.java b/src/com/subgraph/orchid/directory/downloader/HttpConnection.java index 96db9eaa..44d778d3 100644 --- a/src/com/subgraph/orchid/directory/downloader/HttpConnection.java +++ b/src/com/subgraph/orchid/directory/downloader/HttpConnection.java @@ -15,7 +15,6 @@ import com.subgraph.orchid.Router; import com.subgraph.orchid.Stream; -import com.subgraph.orchid.TorException; public class HttpConnection { private final static Charset CHARSET = Charset.forName("ISO-8859-1"); @@ -23,22 +22,29 @@ public class HttpConnection { private final static String HTTP_RESPONSE_REGEX = "HTTP/1\\.(\\d) (\\d+) (.*)"; private final static String CONTENT_LENGTH_HEADER = "Content-Length"; private final static String CONTENT_ENCODING_HEADER = "Content-Encoding"; + private final static String COMPRESSION_SUFFIX = ".z"; private final String hostname; private final Stream stream; private final InputStream input; private final OutputStream output; private final Map headers; + private final boolean useCompression; private int responseCode; private boolean bodyCompressed; private String responseMessage; private ByteBuffer messageBody; public HttpConnection(Stream stream) { + this(stream, true); + } + + public HttpConnection(Stream stream, boolean useCompression) { this.hostname = getHostnameFromStream(stream); this.stream = stream; this.headers = new HashMap(); this.input = stream.getInputStream(); this.output = stream.getOutputStream(); + this.useCompression = useCompression; } private static String getHostnameFromStream(Stream stream) { @@ -59,6 +65,9 @@ public void sendGetRequest(String request) throws IOException { final StringBuilder sb = new StringBuilder(); sb.append("GET "); sb.append(request); + if(useCompression && !request.endsWith(COMPRESSION_SUFFIX)) { + sb.append(COMPRESSION_SUFFIX); + } sb.append(" HTTP/1.0\r\n"); if(hostname != null) { sb.append("Host: "+ hostname +"\r\n"); @@ -78,7 +87,7 @@ public String getHost() { } } - public void readResponse() throws IOException { + public void readResponse() throws IOException, DirectoryRequestFailedException { readStatusLine(); readHeaders(); readBody(); @@ -101,30 +110,29 @@ public void close() { return; } stream.close(); - stream.getCircuit().destroyCircuit(); } - private void readStatusLine() throws IOException { + private void readStatusLine() throws IOException, DirectoryRequestFailedException { final String line = nextResponseLine(); final Pattern p = Pattern.compile(HTTP_RESPONSE_REGEX); final Matcher m = p.matcher(line); if(!m.find() || m.groupCount() != 3) - throw new TorException("Error parsing HTTP response line: "+ line); + throw new DirectoryRequestFailedException("Error parsing HTTP response line: "+ line); try { int n1 = Integer.parseInt(m.group(1)); int n2 = Integer.parseInt(m.group(2)); if( (n1 != 0 && n1 != 1) || (n2 < 100 || n2 >= 600)) - throw new TorException("Failed to parse header: "+ line); + throw new DirectoryRequestFailedException("Failed to parse header: "+ line); responseCode = n2; responseMessage = m.group(3); } catch(NumberFormatException e) { - throw new TorException("Failed to parse header: "+ line); + throw new DirectoryRequestFailedException("Failed to parse header: "+ line); } } - private void readHeaders() throws IOException { + private void readHeaders() throws IOException, DirectoryRequestFailedException { headers.clear(); while(true) { final String line = nextResponseLine(); @@ -132,20 +140,20 @@ private void readHeaders() throws IOException { return; final String[] args = line.split(": ", 2); if(args.length != 2) - throw new TorException("Failed to parse HTTP header: "+ line); + throw new DirectoryRequestFailedException("Failed to parse HTTP header: "+ line); headers.put(args[0], args[1]); } } - private String nextResponseLine() throws IOException { + private String nextResponseLine() throws IOException, DirectoryRequestFailedException { final String line = readInputLine(); if(line == null) { - throw new TorException("Unexpected EOF reading HTTP response"); + throw new DirectoryRequestFailedException("Unexpected EOF reading HTTP response"); } return line; } - private void readBody() throws IOException { + private void readBody() throws IOException, DirectoryRequestFailedException { processContentEncodingHeader(); if(headers.containsKey(CONTENT_LENGTH_HEADER)) { @@ -155,14 +163,14 @@ private void readBody() throws IOException { } } - private void processContentEncodingHeader() { + private void processContentEncodingHeader() throws DirectoryRequestFailedException { final String encoding = headers.get(CONTENT_ENCODING_HEADER); if(encoding == null || encoding.equals("identity")) bodyCompressed = false; else if(encoding.equals("deflate") || encoding.equals("x-deflate")) bodyCompressed = true; else - throw new TorException("Unrecognized content encoding: "+ encoding); + throw new DirectoryRequestFailedException("Unrecognized content encoding: "+ encoding); } private void readBodyFromContentLength() throws IOException { diff --git a/src/com/subgraph/orchid/directory/downloader/MicrodescriptorFetcher.java b/src/com/subgraph/orchid/directory/downloader/MicrodescriptorFetcher.java new file mode 100644 index 00000000..ab407cf9 --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/MicrodescriptorFetcher.java @@ -0,0 +1,44 @@ +package com.subgraph.orchid.directory.downloader; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.subgraph.orchid.RouterMicrodescriptor; +import com.subgraph.orchid.data.HexDigest; +import com.subgraph.orchid.directory.parsing.DocumentParser; + +public class MicrodescriptorFetcher extends DocumentFetcher{ + + private final List fingerprints; + + public MicrodescriptorFetcher(Collection fingerprints) { + this.fingerprints = new ArrayList(fingerprints); + } + + @Override + String getRequestPath() { + return "/tor/micro/d/"+ fingerprintsToRequestString(); + } + + private String fingerprintsToRequestString() { + final StringBuilder sb = new StringBuilder(); + for(HexDigest fp: fingerprints) { + appendFingerprint(sb, fp); + } + return sb.toString(); + } + + private void appendFingerprint(StringBuilder sb, HexDigest fp) { + if(sb.length() > 0) { + sb.append("-"); + } + sb.append(fp.toBase64(true)); + } + + @Override + DocumentParser createParser(ByteBuffer response) { + return PARSER_FACTORY.createRouterMicrodescriptorParser(response); + } +} diff --git a/src/com/subgraph/orchid/directory/downloader/RouterDescriptorFetcher.java b/src/com/subgraph/orchid/directory/downloader/RouterDescriptorFetcher.java new file mode 100644 index 00000000..4b03e88c --- /dev/null +++ b/src/com/subgraph/orchid/directory/downloader/RouterDescriptorFetcher.java @@ -0,0 +1,43 @@ +package com.subgraph.orchid.directory.downloader; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.subgraph.orchid.RouterDescriptor; +import com.subgraph.orchid.data.HexDigest; +import com.subgraph.orchid.directory.parsing.DocumentParser; + +public class RouterDescriptorFetcher extends DocumentFetcher{ + + private final List fingerprints; + + public RouterDescriptorFetcher(Collection fingerprints) { + this.fingerprints = new ArrayList(fingerprints); + } + + @Override + String getRequestPath() { + return "/tor/server/d/"+ fingerprintsToRequestString(); + } + + private String fingerprintsToRequestString() { + final StringBuilder sb = new StringBuilder(); + for(HexDigest fp: fingerprints) { + appendFingerprint(sb, fp); + } + return sb.toString(); + } + private void appendFingerprint(StringBuilder sb, HexDigest fp) { + if(sb.length() > 0) { + sb.append("+"); + } + sb.append(fp.toString()); + } + + @Override + DocumentParser createParser(ByteBuffer response) { + return PARSER_FACTORY.createRouterDescriptorParser(response, true); + } +}