Skip to content
This repository has been archived by the owner on Jan 5, 2022. It is now read-only.

Commit

Permalink
Added resource for getting Import
Browse files Browse the repository at this point in the history 
Updated importService methods to support queries

Scheduler appears broken
  • Loading branch information
Todd Nine committed Feb 11, 2015
1 parent 58f4828 commit 728ca95
Show file tree
Hide file tree
Showing 9 changed files with 161 additions and 121 deletions.
40 changes: 34 additions & 6 deletions stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
View file
Expand Up @@ -17,7 +17,9 @@
package org.apache.usergrid.rest; package org.apache.usergrid.rest;




import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map;


import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
Expand All @@ -34,6 +36,9 @@
import org.apache.usergrid.security.tokens.TokenService; import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory; import org.apache.usergrid.services.ServiceManagerFactory;


import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.core.HttpContext; import com.sun.jersey.api.core.HttpContext;
import com.sun.jersey.api.core.ResourceContext; import com.sun.jersey.api.core.ResourceContext;
import com.sun.jersey.api.view.Viewable; import com.sun.jersey.api.view.Viewable;
Expand All @@ -49,6 +54,11 @@


public abstract class AbstractContextResource { public abstract class AbstractContextResource {


protected static final TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {};
protected static final TypeReference<List<Object>> listTypeReference = new TypeReference<List<Object>>() {};
protected static final ObjectMapper mapper = new ObjectMapper();


protected AbstractContextResource parent; protected AbstractContextResource parent;


@Context @Context
Expand Down Expand Up @@ -130,7 +140,7 @@ public PathSegment getFirstPathSegment( String name ) {




public boolean useReCaptcha() { public boolean useReCaptcha() {
return StringUtils.isNotBlank( properties.getRecaptchaPublic() ) return StringUtils.isNotBlank( properties.getRecaptchaPublic() )
&& StringUtils.isNotBlank( properties.getRecaptchaPrivate() ); && StringUtils.isNotBlank( properties.getRecaptchaPrivate() );
} }


Expand All @@ -139,7 +149,7 @@ public String getReCaptchaHtml() {
if ( !useReCaptcha() ) { if ( !useReCaptcha() ) {
return ""; return "";
} }
ReCaptcha c = ReCaptchaFactory.newSecureReCaptcha( ReCaptcha c = ReCaptchaFactory.newSecureReCaptcha(
properties.getRecaptchaPublic(), properties.getRecaptchaPrivate(), false ); properties.getRecaptchaPublic(), properties.getRecaptchaPrivate(), false );
return c.createRecaptchaHtml( null, null ); return c.createRecaptchaHtml( null, null );
} }
Expand All @@ -152,30 +162,48 @@ public void sendRedirect( String location ) {
} }




public Viewable handleViewable( String template, Object model ) { public Viewable handleViewable( String template, Object model ) {


String className = this.getClass().getName().toLowerCase(); String className = this.getClass().getName().toLowerCase();
String packageName = AbstractContextResource.class.getPackage().getName(); String packageName = AbstractContextResource.class.getPackage().getName();


String template_property = "usergrid.view" + String template_property = "usergrid.view" +
StringUtils.removeEnd( className.toLowerCase(), "resource" ) StringUtils.removeEnd( className.toLowerCase(), "resource" )
.substring( packageName.length() ) + "." + template.toLowerCase(); .substring( packageName.length() ) + "." + template.toLowerCase();


String redirect_url = properties.getProperty( template_property ); String redirect_url = properties.getProperty( template_property );

if ( StringUtils.isNotBlank( redirect_url ) ) { if ( StringUtils.isNotBlank( redirect_url ) ) {
logger.debug("Redirecting to URL: ", redirect_url); logger.debug("Redirecting to URL: ", redirect_url);
sendRedirect( redirect_url ); sendRedirect( redirect_url );
} }
logger.debug("Dispatching to viewable with template: {}", logger.debug("Dispatching to viewable with template: {}",
template, template_property ); template, template_property );


Viewable viewable = new Viewable( template, model, this.getClass() ); Viewable viewable = new Viewable( template, model, this.getClass() );
return viewable; return viewable;
} }





protected ApiResponse createApiResponse() { protected ApiResponse createApiResponse() {
return new ApiResponse( properties ); return new ApiResponse( properties );
} }

/**
* Next three new methods necessary to work around inexplicable problems with EntityHolder.
* This problem happens consistently when you deploy "two-dot-o" to Tomcat:
* https://groups.google.com/forum/#!topic/usergrid/yyAJdmsBfig
*/
protected Object readJsonToObject( String content ) throws IOException {

JsonNode jsonNode = mapper.readTree( content );
Object jsonObject;
if ( jsonNode.isArray() ) {
jsonObject = mapper.readValue( content, listTypeReference );
} else {
jsonObject = mapper.readValue( content, mapTypeReference );
}
return jsonObject;
}
} }
23 changes: 3 additions & 20 deletions stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
View file
Expand Up @@ -90,8 +90,6 @@ public class ServiceResource extends AbstractContextResource {
protected static final Logger LOG = LoggerFactory.getLogger( ServiceResource.class ); protected static final Logger LOG = LoggerFactory.getLogger( ServiceResource.class );
private static final String FILE_FIELD_NAME = "file"; private static final String FILE_FIELD_NAME = "file";


protected static TypeReference<Map<String, Object>> mapTypeReference = new TypeReference<Map<String, Object>>() {};
protected static TypeReference<List<Object>> listTypeReference = new TypeReference<List<Object>>() {};


@Autowired @Autowired
private BinaryStore binaryStore; private BinaryStore binaryStore;
Expand Down Expand Up @@ -328,28 +326,13 @@ else if ( jsonList.get( 0 ) instanceof Map ) {
} }




/**
* Next three new methods necessary to work around inexplicable problems with EntityHolder.
* This problem happens consistently when you deploy "two-dot-o" to Tomcat:
* https://groups.google.com/forum/#!topic/usergrid/yyAJdmsBfig
*/
protected Object readJsonToObject( String content ) throws IOException {
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree( content );
Object jsonObject;
if ( jsonNode.isArray() ) {
jsonObject = mapper.readValue( content, listTypeReference );
} else {
jsonObject = mapper.readValue( content, mapTypeReference );
}
return jsonObject;
}


/** /**
* Necessary to work around inexplicable problems with EntityHolder. * Necessary to work around inexplicable problems with EntityHolder.
* See above. * See above.
*/ */
public JSONWithPadding executePostWithObject( @Context UriInfo ui, Object json, public JSONWithPadding executePostWithObject( @Context UriInfo ui, Object json,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception { @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {


LOG.debug( "ServiceResource.executePostWithMap" ); LOG.debug( "ServiceResource.executePostWithMap" );
Expand Down Expand Up @@ -394,7 +377,7 @@ public JSONWithPadding executePutWithMap( @Context UriInfo ui, Map<String, Objec
@POST @POST
@RequireApplicationAccess @RequireApplicationAccess
@Consumes(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON)
public JSONWithPadding executePost( @Context UriInfo ui, String body, public JSONWithPadding executePost( @Context UriInfo ui, String body,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception { @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {


LOG.debug( "ServiceResource.executePost: body = " + body ); LOG.debug( "ServiceResource.executePost: body = " + body );
Expand Down
59 changes: 20 additions & 39 deletions ...g/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
View file
Expand Up @@ -52,6 +52,7 @@
import org.apache.usergrid.management.importer.ImportService; import org.apache.usergrid.management.importer.ImportService;
import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.entities.Import; import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.index.query.Identifier; import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials; import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
import org.apache.usergrid.rest.AbstractContextResource; import org.apache.usergrid.rest.AbstractContextResource;
Expand Down Expand Up @@ -79,7 +80,7 @@
@Component("org.apache.usergrid.rest.management.organizations.applications.ImportsResource") @Component("org.apache.usergrid.rest.management.organizations.applications.ImportsResource")
@Scope("prototype") @Scope("prototype")
@Produces(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON)
public class ImportsResource extends ServiceResource { public class ImportsResource extends AbstractContextResource {




@Autowired @Autowired
Expand All @@ -100,7 +101,6 @@ public ImportsResource() {
public ImportsResource init( final OrganizationInfo organization, final ApplicationInfo application ){ public ImportsResource init( final OrganizationInfo organization, final ApplicationInfo application ){
this.organization = organization; this.organization = organization;
this.application = application; this.application = application;
services = smf.getServiceManager( application.getId() );
return this; return this;
} }


Expand All @@ -112,13 +112,12 @@ public JSONWithPadding executePost( @Context UriInfo ui, String body,
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception { throws Exception {


LOG.debug( "ServiceResource.executePost: body = " + body );


ApiResponse response = createApiResponse(); ApiResponse response = createApiResponse();




response.setAction( "post" ); response.setAction( "post" );
response.setApplication( services.getApplication() ); response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
response.setParams( ui.getQueryParameters() ); response.setParams( ui.getQueryParameters() );


final Map<String, Object> json = ( Map<String, Object> ) readJsonToObject( body ); final Map<String, Object> json = ( Map<String, Object> ) readJsonToObject( body );
Expand Down Expand Up @@ -159,56 +158,38 @@ public JSONWithPadding executePost( @Context UriInfo ui, String body,
json.put( "organizationId", organization.getUuid() ); json.put( "organizationId", organization.getUuid() );
json.put( "applicationId", application.getId() ); json.put( "applicationId", application.getId() );


Import importEntity = importService.schedule( json ); Import importEntity = importService.schedule( application.getId(), json );


response.setEntities( Collections.<Entity>singletonList( importEntity ) ); response.setEntities( Collections.<Entity>singletonList( importEntity ) );
// }
// catch ( NullPointerException e ) {
//
// response.set
// return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) )
// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) )
// .build();
// }
// catch ( AmazonClientException e ) {
// return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) )
// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) )
// .build();
// }
// catch ( Exception e ) {
// //TODO:throw descriptive error message and or include on in the response
// //TODO:fix below, it doesn't work if there is an exception. Make it look like the
// OauthResponse.
// OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_INTERNAL_SERVER_ERROR )
// .setErrorDescription( e.getMessage() )
// .buildJSONMessage();
// return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType(
// callback ) )
// .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) )
// .build();
// }

// return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();



return new JSONWithPadding( response, callback ); return new JSONWithPadding( response, callback );
} }




@Override @GET
@Path( RootResource.ENTITY_ID_PATH ) @Path( RootResource.ENTITY_ID_PATH )
public AbstractContextResource addIdParameter( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId ) public JSONWithPadding addIdParameter( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId )
throws Exception { throws Exception {


final UUID importId = UUID.fromString( entityId.getPath() );
final Import importEntity = importService.getImport( application.getId(), importId);

if(importEntity == null){
throw new EntityNotFoundException( "could not find import with uuid " + importId );
}

ApiResponse response = createApiResponse();



UUID itemId = UUID.fromString( entityId.getPath() ); response.setAction( "get" );
response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
response.setParams( ui.getQueryParameters() );


addParameter( getServiceParameters(), itemId );


addMatrixParams( getServiceParameters(), ui, entityId ); response.setEntities( Collections.<Entity>singletonList( importEntity ) );


return new JSONWithPadding( response );


return getSubResource( ImportResource.class ).init( itemId );
} }




Expand Down
9 changes: 7 additions & 2 deletions stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
View file
Expand Up @@ -630,12 +630,17 @@ private Entity importCollection() throws Exception {
Entity importEntity = this.management().orgs().organization( org ).app().addToPath( app ).addToPath( "import" ).post( importPayload ); Entity importEntity = this.management().orgs().organization( org ).app().addToPath( app ).addToPath( "import" ).post( importPayload );




Entity importGet = this.management().orgs().organization( org ).app().addToPath( app ).addToPath( "import" ).addToPath( importEntity.getUuid().toString() ).get();


int maxRetries = 120; int maxRetries = 120;
int retries = 0; int retries = 0;


while ( !importGet.get( "state" ).equals( "FINISHED" ) && retries++ < maxRetries ) { while ( retries++ < maxRetries ) {

Entity importGet = this.management().orgs().organization( org ).app().addToPath( app ).addToPath( "import" ).addToPath( importEntity.getUuid().toString() ).get();
if(importGet.get( "state" ).equals( "FINISHED" )){
break;
}

logger.debug("Waiting for import..."); logger.debug("Waiting for import...");
Thread.sleep(1000); Thread.sleep(1000);
} }
Expand Down
2 changes: 1 addition & 1 deletion stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
View file
Expand Up @@ -43,7 +43,7 @@ public class FileImportTracker {
private static final String ERROR_MESSAGE = private static final String ERROR_MESSAGE =
"Failed to import some data. See the import counters and errors."; "Failed to import some data. See the import counters and errors.";


private static final String ERRORS_CONNECTION_NAME = "errors"; public static final String ERRORS_CONNECTION_NAME = "errors";


private final AtomicLong entitiesWritten = new AtomicLong( 0 ); private final AtomicLong entitiesWritten = new AtomicLong( 0 );
private final AtomicLong entitiesFailed = new AtomicLong( 0 ); private final AtomicLong entitiesFailed = new AtomicLong( 0 );
Expand Down
38 changes: 20 additions & 18 deletions stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java
View file
Expand Up @@ -17,31 +17,38 @@


package org.apache.usergrid.management.importer; package org.apache.usergrid.management.importer;



import java.util.Map;
import java.util.UUID;

import org.apache.usergrid.batch.JobExecution; import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.persistence.Results; import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.entities.FailedImportEntity; import org.apache.usergrid.persistence.entities.FailedImportEntity;
import org.apache.usergrid.persistence.entities.FileImport; import org.apache.usergrid.persistence.entities.FileImport;
import org.apache.usergrid.persistence.entities.Import; import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.services.queues.ImportQueueMessage; import org.apache.usergrid.services.queues.ImportQueueMessage;


import java.io.File;
import java.util.ArrayList;
import java.util.Map;
import java.util.UUID;



/** /**
* Performs all functions related to importing. * Performs all functions related to importing.
*/ */
public interface ImportService { public interface ImportService {


enum ImportType { COLLECTION, APPLICATION, ORGANIZATION }

/** /**
* Schedules the import to execute * Schedules the import to execute
*/ */
Import schedule( Map<String, Object> json ) throws Exception; Import schedule( final UUID applicationId, Map<String, Object> json ) throws Exception;

Results getImports(final UUID applicationId, final String cursor);

/**
* Get the import
* @param applicationId
* @param importId
* @return
*/
Import getImport(final UUID applicationId, final UUID importId);


/** /**
* Get the results * Get the results
Expand All @@ -50,15 +57,15 @@ enum ImportType { COLLECTION, APPLICATION, ORGANIZATION }
* @param cursor The cursor used in parsing * @param cursor The cursor used in parsing
* @return * @return
*/ */
Results getFileImports( final UUID importId, final String cursor); Results getFileImports(final UUID applicationId, final UUID importId, final String cursor);


/** /**
* Get the file import * Get the file import
* @param importId * @param importId
* @param fileImportId * @param fileImportId
* @return * @return
*/ */
FileImport getFileImport(final UUID importId, final UUID fileImportId); FileImport getFileImport(final UUID applicationId, final UUID importId, final UUID fileImportId);






Expand All @@ -70,7 +77,7 @@ enum ImportType { COLLECTION, APPLICATION, ORGANIZATION }
* @param cursor The cursor used in parsing * @param cursor The cursor used in parsing
* @return * @return
*/ */
Results getFailedImports( final UUID importId, final UUID fileImportId, final String cursor); Results getFailedImports(final UUID applicationId, final UUID importId, final UUID fileImportId, final String cursor);


/** /**
* Get the failedimport entity from it's parentId * Get the failedimport entity from it's parentId
Expand All @@ -79,7 +86,7 @@ enum ImportType { COLLECTION, APPLICATION, ORGANIZATION }
* @param failedImportId * @param failedImportId
* @return * @return
*/ */
FailedImportEntity getFailedImportEntity(final UUID importId, final UUID fileImportId, final UUID failedImportId); FailedImportEntity getFailedImportEntity(final UUID applicationId, final UUID importId, final UUID fileImportId, final UUID failedImportId);


/** /**
* Perform the import from the external resource * Perform the import from the external resource
Expand All @@ -105,11 +112,6 @@ enum ImportType { COLLECTION, APPLICATION, ORGANIZATION }
*/ */
String getErrorMessage(UUID uuid) throws Exception; String getErrorMessage(UUID uuid) throws Exception;


/**
* @return FileImportEntity
*/
FileImport getFileImportEntity(final ImportQueueMessage importQueueMessage) throws Exception;

/** /**
* @return FileImportEntity * @return FileImportEntity
*/ */
Expand Down

0 comments on commit 728ca95

Please sign in to comment.