Skip to content

Commit

Permalink
re #8924 refactor to use InternetHelper
Browse files Browse the repository at this point in the history
  • Loading branch information
@NickDraper
NickDraper committed Nov 24, 2014
1 parent a34f1ef commit 0facbc0
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 238 deletions.
6 changes: 0 additions & 6 deletions Code/Mantid/Framework/DataHandling/CMakeLists.txt
View file
Expand Up @@ -158,11 +158,8 @@ set ( INC_FILES
inc/MantidDataHandling/DefineGaugeVolume.h
inc/MantidDataHandling/DeleteTableRows.h
inc/MantidDataHandling/DetermineChunking.h
<<<<<<< HEAD
inc/MantidDataHandling/DownloadInstrument.h
=======
inc/MantidDataHandling/DownloadFile.h
>>>>>>> 8450787b233cb81fee1cb7776982d04dc4e0b2a4
inc/MantidDataHandling/ExtractMonitorWorkspace.h
inc/MantidDataHandling/FilterEventsByLogValuePreNexus.h
inc/MantidDataHandling/FindDetectorsInShape.h
Expand Down Expand Up @@ -308,11 +305,8 @@ set ( TEST_FILES
DefineGaugeVolumeTest.h
DeleteTableRowsTest.h
DetermineChunkingTest.h
<<<<<<< HEAD
DownloadInstrumentTest.h
=======
DownloadFileTest.h
>>>>>>> 8450787b233cb81fee1cb7776982d04dc4e0b2a4
ExtractMonitorWorkspaceTest.h
FilterEventsByLogValuePreNexusTest.h
FindDetectorsInShapeTest.h
Expand Down
3 changes: 1 addition & 2 deletions Code/Mantid/Framework/DataHandling/inc/MantidDataHandling/DownloadInstrument.h
View file
Expand Up @@ -52,8 +52,7 @@ namespace DataHandling
void exec();
virtual int doDownloadFile(const std::string& urlFile, const std::string& localFilePath = "",
const StringToStringMap& headers = StringToStringMap());
StringToStringMap updateJsonFile(const std::string& directoryPath, const std::string& filePath);
const std::string mangleFileName(const std::string& filename) const;
StringToStringMap getFileShas(const std::string& directoryPath, const std::string& filePath);
const std::string getDownloadableRepoUrl(const std::string& filename) const;
StringToStringMap processRepository();
std::string getValueOrDefault(const StringToStringMap & mapping,
Expand Down
251 changes: 21 additions & 230 deletions Code/Mantid/Framework/DataHandling/src/DownloadInstrument.cpp
View file
@@ -1,21 +1,13 @@
#include "MantidDataHandling/DownloadInstrument.h"
#include "MantidKernel/ChecksumHelper.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/NetworkProxy.h"
#include "MantidKernel/InternetHelper.h"

// Poco
#include <Poco/DateTimeFormat.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/DateTimeParser.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/Net/AcceptCertificateHandler.h>
#include <Poco/Net/NetException.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
#include <Poco/Net/HTTPSClientSession.h>
#include <Poco/Net/PrivateKeyPassphraseHandler.h>
#include <Poco/Net/SecureStreamSocket.h>
#include <Poco/Net/SSLManager.h>
// Visual Studio complains with the inclusion of Poco/FileStream
// disabling this warning.
#if defined(_WIN32) || defined(_WIN64)
Expand All @@ -30,8 +22,6 @@
#include <Poco/NullStream.h>
#include <stdlib.h>
#endif
#include <Poco/StreamCopier.h>
#include <Poco/URI.h>

// jsoncpp
#include <jsoncpp/json/json.h>
Expand Down Expand Up @@ -163,17 +153,28 @@ namespace Mantid
"e.g. https://api.github.com/repos/mantidproject/mantid/contents/Code/Mantid/instrument.");
}
StringToStringMap fileMap;
if (doDownloadFile(gitHubInstrumentRepoUrl, gitHubJson.toString(),headers) == HTTPResponse::HTTP_NOT_MODIFIED)
try
{
doDownloadFile(gitHubInstrumentRepoUrl, gitHubJson.toString(),headers);
}
catch (Exception::InternetError& ex)
{
//No changes since last time
return fileMap;
if (ex.errorCode() == HTTPResponse::HTTP_NOT_MODIFIED)
{
//No changes since last time
return fileMap;
}
else
{
throw;
}
}

//update local repo files
Poco::Path installRepoFile(localPath, "install.json");
StringToStringMap installShas = updateJsonFile(installPath.toString(), installRepoFile.toString());
StringToStringMap installShas = getFileShas(installPath.toString(), installRepoFile.toString());
Poco::Path localRepoFile(localPath, "local.json");
StringToStringMap localShas = updateJsonFile(localPath.toString(), localRepoFile.toString());
StringToStringMap localShas = getFileShas(localPath.toString(), localRepoFile.toString());

// Parse the server JSON response
Json::Reader reader;
Expand Down Expand Up @@ -232,34 +233,8 @@ namespace Mantid
* @return A map of file names to sha1 values
**/
DownloadInstrument::StringToStringMap
DownloadInstrument::updateJsonFile(const std::string& directoryPath, const std::string& filePath)
DownloadInstrument::getFileShas(const std::string& directoryPath, const std::string& filePath)
{
Json::Value root;
Json::Reader reader;
//check if the file exists
Poco::File catalogFile(filePath);
if (catalogFile.exists() && catalogFile.isFile())
{
std::ifstream catalogStream(filePath, std::ios::in);
if (!reader.parse(catalogStream, root))
{
throw std::runtime_error("Unable to parse JSON file \"" + filePath + "\"");
}
}

// -- File layout --
//
// [
// {
// "name": "ALF_Definition.xml",
// "lastModified": "1900-01-01 00:00:00",
// "path": "/path/to/file.xml"
// "sha": "fff6fe3a23bf1c8ea0692b4a883af99bee26fd3b",
// "size": 625
// },
// ...
// ]

StringToStringMap filesToSha;
try
{
Expand All @@ -269,41 +244,9 @@ namespace Mantid
{
const auto & entryPath = Poco::Path(it->path());
if (entryPath.getExtension() != "xml") continue;

Json::Value & element = root.append(Json::Value());
element["name"] = entryPath.getFileName();
element["lastModified"] = Poco::DateTimeFormatter::format(it->getLastModified(),
Poco::DateTimeFormat::SORTABLE_FORMAT);
element["path"] = entryPath.toString();
std::string sha1 = ChecksumHelper::gitSha1FromFile(entryPath.toString());
element["sha"] = sha1;
element["size"] = static_cast<Json::UInt64>(it->getSize());
// Track sha1
filesToSha.insert(std::make_pair(entryPath.getFileName(), sha1));

// //get current values
// Poco::LocalDateTime dateTime(it->getLastModified());
// size_t entrySize = it->getSize();

// // read previous values
// std::string keyBase = mangleFileName(it->path());
// Json::UInt64 previousSize = pt.get(keyBase + ".size", 0).asUInt64();
// std::string pdtString = pt.get(keyBase + ".lastModified","1900-01-01 00:00:00").asString();
// int tzd(0);
// Poco::DateTime previousDateTime;
// Poco::DateTimeParser::tryParse(Poco::DateTimeFormat::SORTABLE_FORMAT, pdtString, previousDateTime, tzd);
// previousDateTime += Poco::Timespan(1,0); //add a second as milliseconds are truncated off in the file
// std::string prevSha = pt.get(keyBase + ".sha","");

// //update and generate sha if anything has changed
// if ((entrySize != previousSize) || (dateTime > previousDateTime) || (prevSha == ""))
// {

// pt.put(keyBase + ".size", static_cast<Json::UInt64>(entrySize));
// pt.put(keyBase + ".path", entryPath.toString());
// pt.put(keyBase + ".lastModified", Poco::DateTimeFormatter::format(dateTime, Poco::DateTimeFormat::SORTABLE_FORMAT));
// pt.put(keyBase + ".sha", ChecksumHelper::gitSha1FromFile(entryPath.toString()));
// }
}
}
catch (Poco::Exception & ex)
Expand All @@ -319,27 +262,9 @@ namespace Mantid
throw std::runtime_error(ss.str());
}

// Persist file contents
Poco::FileStream fileOut(filePath, std::ios::out);
fileOut << root;

return filesToSha;
}

/** Converts a filename into a valid key for a boost property tree.
* @param filename The filename and extension with or without path
* @returns a mangled filename that is valid for use as a property tree key
**/
const std::string DownloadInstrument::mangleFileName(const std::string& filename) const
{
Poco::Path entryPath(filename);
std::string entryExt = entryPath.getExtension();
std::string entryName = entryPath.getBaseName();
std::stringstream ss;
ss << entryName << "_" << entryExt;
return ss.str();
}

/** Converts a github file page to a downloadable url for the file.
* @param filename a github file page url
* @returns a downloadable url for the file
Expand All @@ -350,20 +275,7 @@ namespace Mantid
}

/** Download a url and fetch it inside the local path given.
This method was initially modelled on doDownloadFile from the ScriptRepositoryImpl
and then converted to handle SSL connections
@param urlFile: Define a valid URL for the file to be downloaded. Eventually, it may give
any valid https path. For example:
url_file = "https://www.google.com"
url_file = "https://mantidweb/repository/README.md"
The result is to connect to the http server, and request the path given.
The answer, will be inserted at the local_file_path.
This calls Kernel/InternetHelper, but is wrapped in this method to allow mocking in the unit tests.
@param localFilePath [optional] : Provide the destination of the file downloaded at the url_file.
the connection and the download was done correctly.
Expand All @@ -377,129 +289,8 @@ namespace Mantid
const StringToStringMap & headers)
{
int retStatus = 0;
g_log.debug() << "DoDownloadFile : " << urlFile << " to file: " << localFilePath << std::endl;

Poco::URI uri(urlFile);
try {
// initialize ssl
Poco::SharedPtr<InvalidCertificateHandler> certificateHandler = \
new AcceptCertificateHandler(true);
// Currently do not use any means of authentication. This should be updated IDS has signed certificate.
const Context::Ptr context = \
new Context(Context::CLIENT_USE, "", "", "", Context::VERIFY_NONE);
// Create a singleton for holding the default context.
// e.g. any future requests to publish are made to this certificate and context.
SSLManager::instance().initializeClient(NULL, certificateHandler, context);
// Create the session
HTTPSClientSession session(uri.getHost(), static_cast<Poco::UInt16>(uri.getPort()));

//set the proxy
if (!m_isProxySet)
{
Mantid::Kernel::NetworkProxy proxyHelper;
m_proxyInfo = proxyHelper.getHttpProxy(uri.getHost());
m_isProxySet = true;
}
if (!m_proxyInfo.emptyProxy())
{
session.setProxyHost(m_proxyInfo.host());
session.setProxyPort(static_cast<Poco::UInt16>(m_proxyInfo.port()));
}

// create a request
HTTPRequest req(HTTPRequest::HTTP_GET, uri.getPathAndQuery(),
HTTPMessage::HTTP_1_1);
req.set("User-Agent","MANTID");
for (auto itHeaders = headers.begin(); itHeaders != headers.end(); ++itHeaders)
{
req.set(itHeaders->first,itHeaders->second);
}
session.sendRequest(req);

HTTPResponse res;
std::istream & rs = session.receiveResponse(res);
retStatus = res.getStatus();
g_log.debug() << "Answer from web: " << res.getStatus() << " "
<< res.getReason() << std::endl;

//get github api rate limit information if available;
int rateLimitRemaining;
DateAndTime rateLimitReset;
try
{
rateLimitRemaining = boost::lexical_cast<int>( res.get("X-RateLimit-Remaining","-1") );
rateLimitReset.set_from_time_t(boost::lexical_cast<int>( res.get("X-RateLimit-Reset","0")));
}
catch( boost::bad_lexical_cast const& )
{
rateLimitRemaining = -1;
}

if (res.getStatus() == HTTPResponse::HTTP_OK)
{
if (localFilePath.empty())
{
// ignore the answer, throw it away
Poco::NullOutputStream null;
Poco::StreamCopier::copyStream(rs, null);
return retStatus;
}
else
{
// copy the file
Poco::FileStream _out(localFilePath);
Poco::StreamCopier::copyStream(rs, _out);
_out.close();
}
}
else if (res.getStatus() == HTTPResponse::HTTP_FOUND)
{
//extract the new location
std::string newLocation = res.get("location","");
if (newLocation != "")
{
retStatus = doDownloadFile(newLocation,localFilePath);
}
}
else if (res.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED)
{
//do nothing - just return the status
}
else if ((res.getStatus() == HTTPResponse::HTTP_FORBIDDEN) && (rateLimitRemaining == 0))
{
throw Exception::InternetError("The Github API rate limit has been reached, try again after " +
rateLimitReset.toSimpleString(),res.getStatus());
}
else
{
std::stringstream info;
std::stringstream ss;
Poco::StreamCopier::copyStream(rs, ss);
if (res.getStatus() == HTTPResponse::HTTP_NOT_FOUND)
info << "Failed to download " << urlFile
<< " because it failed to find this file at the link " << "<a href=\"" << urlFile
<< "\">.\n" << "Hint. Check that link is correct</a>";
else
{
// show the error
info << res.getReason();
info << ss.str();
}
throw Exception::InternetError(info.str() + ss.str(),res.getStatus());
}
} catch (HostNotFoundException & ex)
{
// this exception occurs when the pc is not connected to the internet
std::stringstream info;
info << "Failed to download " << urlFile << " because there is no connection to the host "
<< ex.message() << ".\nHint: Check your connection following this link: <a href=\""
<< urlFile << "\">" << urlFile << "</a> ";
throw Exception::InternetError(info.str() + ex.displayText());

} catch (Poco::Exception & ex)
{
throw Exception::InternetError("Connection and request failed " + ex.displayText());
}
InternetHelper inetHelper;
retStatus = inetHelper.downloadFile(urlFile,localFilePath,headers);
return retStatus;
}

Expand Down

0 comments on commit 0facbc0

Please sign in to comment.