Skip to content

Commit

Permalink
Amigocloud: assorted set of updates (#246, patch by Victor Chernetsky)
Browse files Browse the repository at this point in the history 
* Fixed data field types.
* Output list of datasets if dataset id is not provided.
* Implemented waiting for job to complete on the server. This will improve write to AmigoCloud reliability.
* Updated documentation.



git-svn-id: https://svn.osgeo.org/gdal/trunk@40498 f0d54148-0727-0410-94bb-9a71ac55c965
  • Loading branch information
@rouault
rouault committed Oct 19, 2017
1 parent 2af429a commit 863fdf1
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 13 deletions.
38 changes: 35 additions & 3 deletions gdal/ogr/ogrsf_frmts/amigocloud/drv_amigocloud.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ <h1>AmigoCloud</h1>

(GDAL/OGR &gt;= 2.1)<p>

This driver can connect to the services implementing the AmigoCloud API.
This driver can connect to the AmigoCloud API services.
GDAL/OGR must be built with Curl support in order for the
AmigoCloud driver to be compiled.<p>

Expand All @@ -28,6 +28,15 @@ <h2>Dataset name syntax</h2>
</ul>

If several parameters are specified, they must be separated by a space.<p>
If no datset_id is provided, the driver will print list of available datasets for given project.
<li> For example: <b>"AmigoCloud:1234 datasets"</b>
<pre>
List of available datasets for project id: 1234
| id | name
|-----------|-------------------
| 5551 | points
| 5552 | lines
</pre>

<h2>Configuration options</h2>

Expand Down Expand Up @@ -97,7 +106,30 @@ <h2>Layer creation options</h2>
<h2>Examples</h2>

<li>
Acceccing data from a public table:
Different ways to provide AmigoCloud API token:
<pre>
ogrinfo --config AMIGOCLOUD_API_KEY abcdefghijklmnopqrstuvw -al "AmigoCloud:1234 datasets=987"
ogrinfo -oo AMIGOCLOUD_API_KEY=abcdefghijklmnopqrstuvw -al "AmigoCloud:1234 datasets=987"
env AMIGOCLOUD_API_KEY=abcdefghijklmnopqrstuvw ogrinfo -al "AmigoCloud:1234 datasets=987"
</pre>
<pre>
export AMIGOCLOUD_API_KEY=abcdefghijklmnopqrstuvw
ogrinfo -al "AmigoCloud:1234 datasets=987"
</pre>

<li>
Show list of datasets.
<pre>
$ ogrinfo -ro "AmigoCloud:1234 datasets"
List of available datasets for project id: 1234
| id | name
|-----------|-------------------
| 5551 | points
| 5552 | lines
</pre>

<li>
Accessing data from a public table:
<pre>
ogrinfo -ro "AmigoCloud:1234 datasets=1234,1235"
</pre>
Expand All @@ -106,7 +138,7 @@ <h2>Examples</h2>
<li>
Creating and populating a table from a shapefile:
<pre>
ogr2ogr --config AMIGOCLOUD_API_KEY abcdefghijklmnopqrstuvw -f "AmigoCloud" AmigoCloud:"1234" myshapefile.shp
ogr2ogr --config AMIGOCLOUD_API_KEY abcdefghijklmnopqrstuvw -f AmigoCloud "AmigoCloud:1234" myshapefile.shp
</pre>
<p>

Expand Down
3 changes: 3 additions & 0 deletions gdal/ogr/ogrsf_frmts/amigocloud/ogr_amigocloud.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,9 @@ class OGRAmigoCloudDataSource : public OGRDataSource
OGRGeometry *poSpatialFilter = NULL,
const char *pszDialect = NULL,
bool bRunDeferredActions = false );

bool ListDatasets();
bool waitForJobToFinish(const char* jobId);
};

#endif /* ndef OGR_AMIGOCLOUD_H_INCLUDED */
117 changes: 111 additions & 6 deletions gdal/ogr/ogrsf_frmts/amigocloud/ogramigoclouddatasource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,56 @@ CPLString OGRAMIGOCLOUDGetOptionValue(const char* pszFilename,
return osOptionValue;
}

bool OGRAmigoCloudDataSource::ListDatasets()
{
std::stringstream url;
url << std::string(GetAPIURL()) << "/users/0/projects/" << std::string(GetProjetcId()) << "/datasets/?summary";

json_object* result = RunGET(url.str().c_str());
if( result == NULL ) {
CPLError(CE_Failure, CPLE_AppDefined, "AmigoCloud:get failed.");
return false;
}

if( result != NULL )
{
int type = json_object_get_type(result);
if(type == json_type_object)
{
json_object *poResults = CPL_json_object_object_get(result, "results");
if(poResults != NULL) {
array_list *res = json_object_get_array(poResults);
if(res != NULL) {
CPLprintf("List of available datasets for project id: %s\n", GetProjetcId());
CPLprintf("| id \t | name\n");
CPLprintf("|--------|-------------------\n");
for(int i = 0; i < res->length; i++) {
json_object *ds = (json_object*)array_list_get_idx(res, i);
if(ds!=NULL) {
const char *name = NULL;
int64_t dataset_id = 0;
json_object *poName = CPL_json_object_object_get(ds, "name");
if (poName != NULL) {
name = json_object_get_string(poName);
}
json_object *poId = CPL_json_object_object_get(ds, "id");
if (poId != NULL) {
dataset_id = json_object_get_int64(poId);
}
if (name != NULL) {
std::stringstream str;
str << "| " << dataset_id << "\t | " << name;
CPLprintf("%s\n", str.str().c_str());
} }
}
}
}
}
json_object_put(result);
}
return true;
}

/************************************************************************/
/* Open() */
/************************************************************************/
Expand Down Expand Up @@ -161,15 +211,18 @@ int OGRAmigoCloudDataSource::Open( const char * pszFilename,
}
}

osAPIKey = CSLFetchNameValueDef(papszOpenOptionsIn, "API_KEY",
osAPIKey = CSLFetchNameValueDef(papszOpenOptionsIn, "AMIGOCLOUD_API_KEY",
CPLGetConfigOption("AMIGOCLOUD_API_KEY", ""));

if (osAPIKey.empty())
{
osAPIKey = OGRAMIGOCLOUDGetOptionValue(pszFilename, "API_KEY");
osAPIKey = OGRAMIGOCLOUDGetOptionValue(pszFilename, "AMIGOCLOUD_API_KEY");
}
if (osAPIKey.empty())
{
CPLError(CE_Failure, CPLE_AppDefined, "AMIGOCLOUD_API_KEY is not defined.\n");
return FALSE;
}

CPLString osDatasets = OGRAMIGOCLOUDGetOptionValue(pszFilename, "datasets");

bUseHTTPS = CPLTestBool(CPLGetConfigOption("AMIGOCLOUD_HTTPS", "YES"));

Expand All @@ -190,6 +243,7 @@ int OGRAmigoCloudDataSource::Open( const char * pszFilename,
if( osCurrentSchema.empty() )
return FALSE;

CPLString osDatasets = OGRAMIGOCLOUDGetOptionValue(pszFilename, "datasets");
if (!osDatasets.empty())
{
char** papszTables = CSLTokenizeString2(osDatasets, ",", 0);
Expand All @@ -201,6 +255,11 @@ int OGRAmigoCloudDataSource::Open( const char * pszFilename,
}
CSLDestroy(papszTables);
return TRUE;
} else {
// If 'datasets' word is in the filename, but no datasets specified,
// print the list of available datasets
if(std::string(pszFilename).find("datasets") != std::string::npos)
ListDatasets();
}

return TRUE;
Expand Down Expand Up @@ -453,6 +512,13 @@ json_object* OGRAmigoCloudDataSource::RunPOST(const char*pszURL, const char *psz
return NULL;
}
}
json_object* poJob = CPL_json_object_object_get(poObj, "job");
if (poJob != NULL) {
const char *job = json_object_get_string(poJob);
if (job != NULL) {
waitForJobToFinish(job);
}
}
}
else
{
Expand All @@ -464,6 +530,41 @@ json_object* OGRAmigoCloudDataSource::RunPOST(const char*pszURL, const char *psz
return poObj;
}

bool OGRAmigoCloudDataSource::waitForJobToFinish(const char* jobId)
{
std::stringstream url;
url << std::string(GetAPIURL()) << "/me/jobs/" << std::string(jobId);

bool done = false;
int count = 0;
while (!done && count<5) {
count++;
json_object *result = RunGET(url.str().c_str());
if (result == NULL) {
CPLError(CE_Failure, CPLE_AppDefined, "AmigoCloud:get failed.");
return false;
}

if (result != NULL) {
int type = json_object_get_type(result);
if (type == json_type_object) {
json_object *poStatus = CPL_json_object_object_get(result, "status");
const char *status = json_object_get_string(poStatus);
if (status != NULL) {
if (std::string(status) == "SUCCESS") {
return true;
} else if (std::string(status) == "FAILURE") {
return false;
}
}
}
}
CPLSleep(1.0); // Sleep 1 sec.
}
return false;
}


/************************************************************************/
/* RunDELETE() */
/************************************************************************/
Expand Down Expand Up @@ -566,13 +667,17 @@ json_object* OGRAmigoCloudDataSource::RunGET(const char*pszURL)
/* -------------------------------------------------------------------- */
if( !osAPIKey.empty() )
{
osURL += "?token=";
if(osURL.find("?") == std::string::npos)
osURL += "?token=";
else
osURL += "&token=";
osURL += osAPIKey;
}

CPLHTTPResult * psResult = CPLHTTPFetch( osURL.c_str(), NULL);
if( psResult == NULL )
if( psResult == NULL ) {
return NULL;
}

if (psResult->pszContentType &&
strncmp(psResult->pszContentType, "text/html", 9) == 0)
Expand Down
3 changes: 1 addition & 2 deletions gdal/ogr/ogrsf_frmts/amigocloud/ogramigoclouddriver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ static GDALDataset *OGRAmigoCloudDriverCreate( const char * pszName,
/************************************************************************/

void RegisterOGRAmigoCloud()

{
if( GDALGetDriverByName( "AmigoCloud" ) != NULL )
return;
Expand All @@ -108,7 +107,7 @@ void RegisterOGRAmigoCloud()

poDriver->SetMetadataItem( GDAL_DMD_OPENOPTIONLIST,
"<OpenOptionList>"
" <Option name='API_KEY' type='string' description='Account API key'/>"
" <Option name='AMIGOCLOUD_API_KEY' type='string' description='Account API key'/>"
" <Option name='PROJECTID' type='string' description='Project id' required='true'/>"
" <Option name='BATCH_INSERT' type='boolean' description='Whether to group features to be inserted in a batch' default='YES'/>"
"</OpenOptionList>");
Expand Down
15 changes: 13 additions & 2 deletions gdal/ogr/ogrsf_frmts/amigocloud/ogramigocloudlayer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,18 +362,29 @@ void OGRAmigoCloudLayer::EstablishLayerDefn(const char* pszLayerName,
}
if(!fieldName.empty() && !fieldType.empty())
{

if(EQUAL(fieldType.c_str(), "string") ||
EQUAL(fieldType.c_str(), "unknown(19)") /* name */ )
{
OGRFieldDefn oFieldDefn(fieldName.c_str(), OFTString);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
else if(EQUAL(fieldType.c_str(), "number"))
else if(EQUAL(fieldType.c_str(), "number") ||
EQUAL(fieldType.c_str(), "float") ||
EQUAL(fieldType.c_str(), "real"))
{
OGRFieldDefn oFieldDefn(fieldName.c_str(), OFTReal);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
else if(EQUAL(fieldType.c_str(), "integer"))
{
OGRFieldDefn oFieldDefn(fieldName.c_str(), OFTInteger);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
else if(EQUAL(fieldType.c_str(), "bigint"))
{
OGRFieldDefn oFieldDefn(fieldName.c_str(), OFTInteger64);
poFeatureDefn->AddFieldDefn(&oFieldDefn);
}
else if(EQUAL(fieldType.c_str(), "date"))
{
if(!EQUAL(fieldName.c_str(), "created_at") &&
Expand Down

0 comments on commit 863fdf1

Please sign in to comment.