Skip to content

Commit

Permalink
maximum number of attachments may exceed during property transfer
Browse files Browse the repository at this point in the history 
During the transfer of a property the maximum number of allowed attachments can exceed temporarily. We should make sure, that old attachments are removed from the Webservice **before** any new attachments are uploaded to the Webservice.
  • Loading branch information
@pinhead84
pinhead84 committed Jan 26, 2015
1 parent 53f459d commit 297e9b3
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 67 deletions.
11 changes: 11 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,17 @@ Changelog for OpenEstate-IS24-REST
-------------------------------


0.2.1 (not released yet)
------------------------

### bugfixes

- During the transfer of a property the maximum number of allowed attachments
can exceed temporarily. We should make sure, that old attachments are
removed from the Webservice **before** any new attachments are uploaded to
the Webservice.


0.2 (27 Dec 2014)
-----------------

Expand Down
202 changes: 135 additions & 67 deletions OpenEstate-IS24-REST/src/main/java/org/openestate/is24/restapi/utils/ExportHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* Copyright 2014-2015 OpenEstate.org.
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -966,14 +966,17 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont
{
Long is24AttachmentId = attachment.getId();
String externalAttachmentId = StringUtils.trimToNull( attachment.getExternalId() );

// Anhang übernehmen, wenn ein Hashwert hinterlegt ist
// und dieser noch nicht übernommen wurde
if (externalAttachmentId!=null && !oldIs24Attachments.containsKey( externalAttachmentId ))
{
//LOGGER.debug( "> found old attachment #" + is24AttachmentId + " / " + externalAttachmentId + " / " + externalAttachmentId.length() );
oldIs24Attachments.put( externalAttachmentId, attachment );
continue;
}

// alten Anhang ohne externe ID entfernen
// alten Anhang entfernen
try
{
//LOGGER.debug( "> removing old attachment #" + is24AttachmentId + " without external id" );
Expand Down Expand Up @@ -1012,24 +1015,13 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont

else
{
// Anhänge aus dem Exportverzeichnis der Immobilie ermitteln
// Anhänge zur Übertragung ermitteln und zugehörige Hash-Werte berechnen
List<String> attachmentHashes = new ArrayList<String>();
Map<Integer,Long> attachmentsOrder = new TreeMap<Integer,Long>();
for (String attachmentPos : this.pool.getObjectAttachmentIds( poolObjectId ))
Map<String,Attachment> attachments = new HashMap<String, Attachment>();
Map<String,File> attachmentFiles = new HashMap<String, File>();
for (String attachmentKey : this.pool.getObjectAttachmentIds( poolObjectId ))
{
int pos;
try
{
pos = Math.abs( Integer.parseInt( attachmentPos ) );
}
catch (NumberFormatException ex)
{
LOGGER.warn( "Can't read attachment position!" );
LOGGER.warn( "> " + ex.getLocalizedMessage(), ex );
pos = 999;
}

Attachment is24Attachment = this.pool.getObjectAttachment( poolObjectId, attachmentPos );
Attachment is24Attachment = this.pool.getObjectAttachment( poolObjectId, attachmentKey );
if (is24Attachment==null)
{
LOGGER.error( "Can't read the XML file for attachment!" );
Expand All @@ -1039,7 +1031,7 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont

// Fortschritt protokollieren
this.addProgress(
this.pool.getObjectAttachmentSize( poolObjectId, attachmentPos ) );
this.pool.getObjectAttachmentSize( poolObjectId, attachmentKey ) );
continue;
}

Expand All @@ -1052,13 +1044,128 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont
URL url = link.getUrl();
String externalAttachmentId = (url!=null)?
DigestUtils.sha1Hex( is24ObjectId + "-" + url.toString() ):
DigestUtils.sha1Hex( is24ObjectId + "-" + attachmentPos );
DigestUtils.sha1Hex( is24ObjectId + "-" + attachmentKey );

// Sicherstellen, dass der gleiche Anhang nicht mehrfach hochgeladen wird
if (attachmentHashes.contains( externalAttachmentId )) continue;
attachmentHashes.add( externalAttachmentId );

link.setExternalId( externalAttachmentId );
}

// Anhang als Datei verarbeiten
else
{
// Datei ermitteln
File attachFile = this.pool.getObjectAttachmentFile( poolObjectId, is24Attachment );

// ggf. Datei herunterladen, wenn noch nicht im Pool hinterlegt
if (attachFile==null)
{
URL attachUrl = this.pool.getObjectAttachmentURL( poolObjectId, attachmentKey );
if (attachUrl!=null)
{
try
{
attachFile = doDownloadFile( attachUrl );
}
catch (Exception ex)
{
LOGGER.warn( "Can't download file from URL: " + attachUrl );
LOGGER.warn( "> " + ex.getLocalizedMessage(), ex );
}
}
}

if (attachFile==null || !attachFile.isFile())
{
LOGGER.error( "Can't find file for attachment!" );
this.putObjectMessage(
externalObjectId, ExportMessage.Code.OBJECT_ATTACHMENT_NOT_SAVED,
"Can't find file for attachment!" );

// Fortschritt protokollieren
this.addProgress(
this.pool.getObjectAttachmentSize( poolObjectId, attachmentKey ) );

continue;
}

// Datei des Anhangs vormerken
attachmentFiles.put( attachmentKey, attachFile.getAbsoluteFile() );

// Hashwert zur Identifizierung des Anhangs errechnen
final String externalAttachmentId;
InputStream input = null;
try
{
input = new FileInputStream( attachFile );
String attachFileHash = DigestUtils.sha1Hex( input );
externalAttachmentId = DigestUtils.sha1Hex( is24ObjectId + "-" + attachFileHash );

// Sicherstellen, dass der gleiche Anhang nicht mehrfach hochgeladen wird
if (attachmentHashes.contains( externalAttachmentId )) continue;
attachmentHashes.add( externalAttachmentId );

is24Attachment.setExternalId( externalAttachmentId );
}
finally
{
IOUtils.closeQuietly( input );
}
}

attachments.put( attachmentKey, is24Attachment );
}

// alte Anhänge entfernen
String[] oldIs24AttachmentIds = oldIs24Attachments.keySet().toArray( new String[oldIs24Attachments.size()] );
for (String oldIs24AttachmentId : oldIs24AttachmentIds)
{
if (attachmentHashes.contains( oldIs24AttachmentId )) continue;
Attachment is24Attachment = oldIs24Attachments.remove( oldIs24AttachmentId );
Long is24AttachmentId = is24Attachment.getId();
try
{
//LOGGER.debug( "> removing old attachment #" + is24AttachmentId );
ImportExport.AttachmentService.deleteById(
this.client, externalObjectId, is24AttachmentId );
}
catch (RequestFailedException ex)
{
LOGGER.error( "Can't remove old attachment (" + is24AttachmentId + ") "
+ "of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
logMessagesAsError( ex.responseMessages );
LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
this.putObjectMessage(
externalObjectId, ExportMessage.Code.OBJECT_OLD_ATTACHMENT_NOT_REMOVED, ex.responseMessages );
}
}

// Anhänge aus dem Exportverzeichnis der Immobilie ermitteln
Map<Integer,Long> attachmentsOrder = new TreeMap<Integer,Long>();
for (Map.Entry<String,Attachment> entry : attachments.entrySet())
{
final String attachmentKey = entry.getKey();
final Attachment is24Attachment = entry.getValue();
final String externalAttachmentId = is24Attachment.getExternalId();

int pos;
try
{
pos = Math.abs( Integer.parseInt( attachmentKey ) );
}
catch (NumberFormatException ex)
{
LOGGER.warn( "Can't read attachment position!" );
LOGGER.warn( "> " + ex.getLocalizedMessage(), ex );
pos = 999;
}

// Anhang als Web-Link verarbeiten
if (is24Attachment instanceof Link)
{
Link link = (Link) is24Attachment;
try
{
// zuvor gespeicherten Web-Link mit gleichem Hashwert aktualisieren
Expand All @@ -1069,7 +1176,7 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont
//LOGGER.debug( "> updating attached link #" + is24AttachmentId );
//LOGGER.debug( ">> " + externalAttachmentId + " / " + externalAttachmentId.length() );
ImportExport.AttachmentService.putById( client,
is24ObjectId, is24AttachmentId, is24Attachment );
is24ObjectId, is24AttachmentId, link );
oldIs24Attachments.remove( externalAttachmentId );
}
// neuen Web-Link erzeugen
Expand All @@ -1092,32 +1199,13 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont

// Fortschritt protokollieren
this.addProgress(
this.pool.getObjectAttachmentSize( poolObjectId, attachmentPos ) );
this.pool.getObjectAttachmentSize( poolObjectId, attachmentKey ) );

continue;
}

// Datei ermitteln
File attachFile = this.pool.getObjectAttachmentFile( poolObjectId, is24Attachment );

// ggf. Datei herunterladen, wenn noch nicht im Pool hinterlegt
if (attachFile==null)
{
URL attachUrl = this.pool.getObjectAttachmentURL( poolObjectId, attachmentPos );
if (attachUrl!=null)
{
try
{
attachFile = doDownloadFile( attachUrl );
}
catch (Exception ex)
{
LOGGER.warn( "Can't download file from URL: " + attachUrl );
LOGGER.warn( "> " + ex.getLocalizedMessage(), ex );
}
}
}

File attachFile = attachmentFiles.get( attachmentKey );
if (attachFile==null || !attachFile.isFile())
{
LOGGER.error( "Can't find file for attachment!" );
Expand All @@ -1127,7 +1215,7 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont

// Fortschritt protokollieren
this.addProgress(
this.pool.getObjectAttachmentSize( poolObjectId, attachmentPos ) );
this.pool.getObjectAttachmentSize( poolObjectId, attachmentKey ) );

continue;
}
Expand All @@ -1136,26 +1224,6 @@ else if (externalContactId!=null && !this.savedContactIds.contains( externalCont
final String attachFileName = attachFile.getName();
final long attachFileSize = attachFile.length();

// Hashwert zur Identifizierung des Anhangs errechnen
final String externalAttachmentId;
InputStream input = null;
try
{
input = new FileInputStream( attachFile );
String attachFileHash = DigestUtils.sha1Hex( input );
externalAttachmentId = DigestUtils.sha1Hex( is24ObjectId + "-" + attachFileHash );

// Sicherstellen, dass der gleiche Anhang nicht mehrfach hochgeladen wird
if (attachmentHashes.contains( externalAttachmentId )) continue;
attachmentHashes.add( externalAttachmentId );

is24Attachment.setExternalId( externalAttachmentId );
}
finally
{
IOUtils.closeQuietly( input );
}

InputStream attachFileInput = null;
try
{
Expand Down Expand Up @@ -1252,23 +1320,23 @@ else if (attachFileName.toLowerCase().endsWith( ".gif" ))

// Fortschritt protokollieren
this.addProgress(
this.pool.getObjectAttachmentSize( poolObjectId, attachmentPos ) + attachFileSize );
this.pool.getObjectAttachmentSize( poolObjectId, attachmentKey ) + attachFileSize );
}
}

// alte / nicht aktualisierte Anhänge entfernen
// nicht aktualisierte Anhänge entfernen
for (Attachment is24Attachment : oldIs24Attachments.values())
{
Long is24AttachmentId = is24Attachment.getId();
try
{
//LOGGER.debug( "> removing old attachment #" + is24AttachmentId );
//LOGGER.debug( "> removing untouched attachment #" + is24AttachmentId );
ImportExport.AttachmentService.deleteById(
this.client, externalObjectId, is24AttachmentId );
}
catch (RequestFailedException ex)
{
LOGGER.error( "Can't remove old attachment (" + is24AttachmentId + ") "
LOGGER.error( "Can't remove untouched attachment (" + is24AttachmentId + ") "
+ "of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
logMessagesAsError( ex.responseMessages );
LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
Expand Down

0 comments on commit 297e9b3

Please sign in to comment.