Skip to content
This repository has been archived by the owner on Aug 30, 2024. It is now read-only.

Commit

Permalink
merging in walrus/sc communication changes.
Browse files Browse the repository at this point in the history
  • Loading branch information
Neil committed Jul 14, 2009
2 parents 4ad479a + bee1409 commit be15a9a
Show file tree
Hide file tree
Showing 13 changed files with 107 additions and 504 deletions.
3 changes: 2 additions & 1 deletion clc/eucalyptus.ipr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
<properties />
<executeOn event="beforeRun" target="install" />
<executeOn event="beforeRun" target="install" runConfigurationType="JUnit" runConfigurationName="DNSControlTest" />
<executeOn event="beforeRun" target="install" runConfigurationType="Remote" runConfigurationName="Unnamed" />
</buildFile>
</component>
<component name="BuildJarProjectSettings">
Expand Down Expand Up @@ -350,7 +351,7 @@
</component>
<component name="ProjectFileVersion" converted="true" />
<component name="ProjectKey">
<option name="state" value="project:///disk1/eucalyptus/src/main/clc/eucalyptus.ipr" />
<option name="state" value="project:///disk1/eucalyptus/src/main-walrus-sc-multicluster/clc/eucalyptus.ipr" />
</component>
<component name="ProjectModuleManager">
<modules>
Expand Down
1 change: 0 additions & 1 deletion clc/modules/core/resources/META-INF/persistence.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
<class>edu.ucsb.eucalyptus.cloud.entities.LVMVolumeInfo</class>
<class>edu.ucsb.eucalyptus.cloud.entities.LVMMetaInfo</class>
<class>edu.ucsb.eucalyptus.cloud.entities.WalrusSnapshotInfo</class>
<class>edu.ucsb.eucalyptus.cloud.entities.WalrusSnapshotSet</class>
<class>edu.ucsb.eucalyptus.cloud.entities.Address</class>
<class>edu.ucsb.eucalyptus.cloud.entities.UserGroupInfo</class>
<class>edu.ucsb.eucalyptus.cloud.entities.ProductCode</class>
Expand Down
42 changes: 6 additions & 36 deletions clc/modules/core/src/edu/ucsb/eucalyptus/cloud/entities/WalrusSnapshotInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,8 @@ public class WalrusSnapshotInfo {
private Long id = -1l;
@Column(name ="snapshot_name")
private String snapshotId;
@Column(name = "snapshotset_id")
private String snapshotSetId;
@Column(name ="snapshot_vgname")
private String vgName;
@Column(name ="snapshot_lvname")
private String lvName;
@Column(name ="transferred")
private Boolean transferred;
@Column(name = "snapshot_bucket")
private String snapshotBucket;
@Column(name = "size")
private Integer size;

Expand All @@ -75,36 +69,12 @@ public void setSnapshotId(String snapshotId) {
this.snapshotId = snapshotId;
}

public String getSnapshotSetId() {
return snapshotSetId;
public String getSnapshotBucket() {
return snapshotBucket;
}

public void setSnapshotSetId(String snapshotSetId) {
this.snapshotSetId = snapshotSetId;
}

public String getVgName() {
return vgName;
}

public void setVgName(String vgName) {
this.vgName = vgName;
}

public String getLvName() {
return lvName;
}

public void setLvName(String lvName) {
this.lvName = lvName;
}

public Boolean getTransferred() {
return transferred;
}

public void setTransferred(Boolean transferred) {
this.transferred = transferred;
public void setSnapshotBucket(String snapshotBucket) {
this.snapshotBucket = snapshotBucket;
}

public Integer getSize() {
Expand Down
87 changes: 0 additions & 87 deletions clc/modules/core/src/edu/ucsb/eucalyptus/cloud/entities/WalrusSnapshotSet.java
View file

This file was deleted.

2 changes: 1 addition & 1 deletion clc/modules/core/src/edu/ucsb/eucalyptus/util/WalrusProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ public enum GetOptionalParameters {
}

public enum StorageOperations {
StoreSnapshot, DeleteWalrusSnapshot, GetSnapshotInfo, GetVolume
StoreSnapshot, DeleteWalrusSnapshot, GetWalrusSnapshot
}

public enum InfoOperations {
Expand Down
4 changes: 2 additions & 2 deletions clc/modules/msgs/resources/s3-2006-03-01-binding.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,11 +398,11 @@
<structure map-as="edu.ucsb.eucalyptus.msgs.WalrusDataResponseType"/>
</mapping>

<mapping name="GetVolume" class="edu.ucsb.eucalyptus.msgs.GetVolumeType">
<mapping name="GetWalrusSnapshot" class="edu.ucsb.eucalyptus.msgs.GetWalrusSnapshotType">
<structure map-as="edu.ucsb.eucalyptus.msgs.WalrusDataRequestType"/>
</mapping>

<mapping name="GetVolumeResponse" class="edu.ucsb.eucalyptus.msgs.GetVolumeResponseType">
<mapping name="GetWalrusSnapshotResponse" class="edu.ucsb.eucalyptus.msgs.GetWalrusSnapshotResponseType">
<structure map-as="edu.ucsb.eucalyptus.msgs.WalrusDataResponseType"/>
</mapping>

Expand Down
4 changes: 2 additions & 2 deletions clc/modules/msgs/src/edu/ucsb/eucalyptus/msgs/Walrus.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,8 +544,8 @@ public class GetSnapshotInfoResponseType extends WalrusResponseType {
ArrayList<String> snapshotSet = new ArrayList<String>();
}

public class GetVolumeType extends WalrusDataRequestType {
public class GetWalrusSnapshotType extends WalrusDataRequestType {
}

public class GetVolumeResponseType extends WalrusDataResponseType {
public class GetWalrusSnapshotResponseType extends WalrusDataResponseType {
}
95 changes: 6 additions & 89 deletions clc/modules/storage-controller/src/edu/ucsb/eucalyptus/cloud/ws/BlockStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ private String getVolume(String volumeId, String snapshotBucket, String snapshot
String volumePath = StorageProperties.storageRootDirectory + "/" + volumeId + Hashes.getRandom(10);
File file = new File(volumePath);
if(!file.exists()) {
HttpReader volumeReader = new HttpReader(walrusSnapshotPath, null, file, "GetVolume", "", true);
HttpReader volumeReader = new HttpReader(walrusSnapshotPath, null, file, "GetWalrusSnapshot", "", true);
volumeReader.run();
} else {
throw new EucalyptusCloudException("volume file already exists");
Expand All @@ -570,27 +570,6 @@ private String getVolume(String volumeId, String snapshotBucket, String snapshot
return volumePath;
}

private List<String> getSnapshots(String snapshotBucket, String snapshotId, List<String> snapshotFileNames) {
String walrusSnapshotPath = snapshotBucket + "/" + snapshotId;
HttpReader reader = new HttpReader(walrusSnapshotPath, null, null, "GetSnapshotInfo", "");
String snapshotDescription = reader.getResponseAsString();
XMLParser parser = new XMLParser(snapshotDescription);
//read the list of snapshots and issue requests to walrus to get all deltas
String bucketName = parser.getValue("/GetSnapshotInfoResponse/Bucket");
List<String> snapshotSet = parser.getValues("/GetSnapshotInfoResponse/snapshotId");
for(String snapshot: snapshotSet) {
walrusSnapshotPath = bucketName + "/" + snapshot;
String snapshotPath = StorageProperties.storageRootDirectory + "/" + snapshot + Hashes.getRandom(10);
snapshotFileNames.add(snapshotPath);
File file = new File(snapshotPath);
HttpReader snapshotReader = new HttpReader(walrusSnapshotPath, null, file, "GetSnapshot", "");
snapshotReader.run();
}
return snapshotSet;
}



public void GetSnapshots(String volumeId, String snapshotSetName, String snapshotId) throws EucalyptusCloudException {
BlockStorageChecker.checkWalrusConnection();
String volumePath = getVolume(volumeId, snapshotSetName, snapshotId);
Expand Down Expand Up @@ -687,85 +666,23 @@ public void run() {
return;
} else {
List<String> returnValues = blockManager.prepareForTransfer(volumeId, snapshotId);
volumeFileName = returnValues.get(0);
snapshotFileName = returnValues.get(1);
EntityWrapper<VolumeInfo> db = new EntityWrapper<VolumeInfo>();
VolumeInfo volumeInfo = new VolumeInfo(volumeId);
List <VolumeInfo> volumeInfos = db.query(volumeInfo);

boolean shouldTransferVolume = false;
if(volumeInfos.size() > 0) {
VolumeInfo foundVolumeInfo = volumeInfos.get(0);
if(!foundVolumeInfo.getTransferred()) {
//transfer volume to Walrus
foundVolumeInfo.setVolumeBucket(volumeBucket);
shouldTransferVolume = true;
}
volumeBucket = foundVolumeInfo.getVolumeBucket();
db.commit();
EntityWrapper<SnapshotInfo> db2 = new EntityWrapper<SnapshotInfo>();
SnapshotInfo snapshotInfo = new SnapshotInfo(snapshotId);
SnapshotInfo foundSnapshotInfo = db2.getUnique(snapshotInfo);
if(foundSnapshotInfo != null) {
transferSnapshot(shouldTransferVolume);
}
db2.commit();
} else {
db.rollback();
throw new EucalyptusCloudException("Volume not found " + volumeId);
}
snapshotFileName = returnValues.get(0);
transferSnapshot();
}
} catch(EucalyptusCloudException ex) {
LOG.error(ex);
}
}

private void transferSnapshot(boolean shouldTransferVolume) {
private void transferSnapshot() {
long size = 0;

File volumeFile = new File(volumeFileName);
File snapshotFile = new File(snapshotFileName);

assert(snapshotFile.exists() && volumeFile.exists());
size += shouldTransferVolume ? snapshotFile.length() + volumeFile.length() : snapshotFile.length();
assert(snapshotFile.exists());
size += snapshotFile.length();
SnapshotProgressCallback callback = new SnapshotProgressCallback(snapshotId, size, StorageProperties.TRANSFER_CHUNK_SIZE);
Map<String, String> httpParamaters = new HashMap<String, String>();
HttpWriter httpWriter;
if(shouldTransferVolume) {
try {
List<String> returnValues = blockManager.getSnapshotValues(volumeId);
if(returnValues.size() > 0) {
httpParamaters.put("SnapshotVgName", returnValues.get(0));
httpParamaters.put("SnapshotLvName", returnValues.get(1));
}
} catch(Exception ex) {
LOG.error(ex);
}
httpWriter = new HttpWriter("PUT", volumeFile, callback, volumeBucket, volumeId, "StoreSnapshot", null, httpParamaters);
try {
httpWriter.run();
EntityWrapper<VolumeInfo>db = new EntityWrapper<VolumeInfo>();
VolumeInfo volumeInfo = new VolumeInfo(volumeId);
List <VolumeInfo> volumeInfos = db.query(volumeInfo);
if(volumeInfos.size() > 0) {
VolumeInfo volInfo = volumeInfos.get(0);
volInfo.setTransferred(true);
}
db.commit();
} catch(Exception ex) {
LOG.error(ex);
return;
}
}
try {
List<String> returnValues = blockManager.getSnapshotValues(snapshotId);
if(returnValues.size() > 0) {
httpParamaters.put("SnapshotVgName", returnValues.get(0));
httpParamaters.put("SnapshotLvName", returnValues.get(1));
}
} catch(Exception ex) {
LOG.error(ex);
}
httpWriter = new HttpWriter("PUT", snapshotFile, callback, volumeBucket, snapshotId, "StoreSnapshot", null, httpParamaters, true);
try {
httpWriter.run();
Expand Down
24 changes: 4 additions & 20 deletions clc/modules/storage-controller/src/edu/ucsb/eucalyptus/storage/LVM2Manager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public class LVM2Manager implements LogicalStorageManager {
public static String hostName = "localhost";
public static final int MAX_LOOP_DEVICES = 256;
public static final int MAX_MINOR_NUMBER = 16;
private static final String blockSize = "1M";
private static final String blockSize = "1M";
public static final String EUCA_ROOT_WRAPPER = "/usr/lib/eucalyptus/euca_rootwrap";
public static final String EUCA_VAR_RUN_PATH = "/var/run/eucalyptus";
private static final String CONFIG_FILE_PATH = "/etc/eucalyptus/eucalyptus.conf";
Expand Down Expand Up @@ -111,7 +111,7 @@ private String getLvmVersion() {
}

private String findFreeLoopback() {
return SystemUtil.run(new String[]{eucaHome + EUCA_ROOT_WRAPPER, "losetup", "-f"}).replaceAll("\n", "");
return SystemUtil.run(new String[]{eucaHome + EUCA_ROOT_WRAPPER, "losetup", "-f"}).replaceAll("\n", "");
}

private String getLoopback(String loDevName) {
Expand Down Expand Up @@ -814,29 +814,13 @@ public List<String> createSnapshot(String volumeId, String snapshotId) throws Eu

public List<String> prepareForTransfer(String volumeId, String snapshotId) throws EucalyptusCloudException {
EntityWrapper<LVMVolumeInfo> db = new EntityWrapper<LVMVolumeInfo>();
LVMVolumeInfo lvmVolumeInfo = new LVMVolumeInfo(volumeId);
LVMVolumeInfo lvmVolumeInfo = new LVMVolumeInfo(snapshotId);
LVMVolumeInfo foundLVMVolumeInfo = db.getUnique(lvmVolumeInfo);
ArrayList<String> returnValues = new ArrayList<String>();

if(foundLVMVolumeInfo != null) {

returnValues.add(StorageProperties.storageRootDirectory + PATH_SEPARATOR + foundLVMVolumeInfo.getVolumeId());
String dmDeviceName = foundLVMVolumeInfo.getVgName().replaceAll("-", "--") + "-" + foundLVMVolumeInfo.getLvName().replaceAll("-", "--");
lvmVolumeInfo = new LVMVolumeInfo(snapshotId);
foundLVMVolumeInfo = db.getUnique(lvmVolumeInfo);
if(foundLVMVolumeInfo != null) {
String snapshotRawFileName = StorageProperties.storageRootDirectory + PATH_SEPARATOR + foundLVMVolumeInfo.getVolumeId();
String dupSnapshotDeltaFileName = snapshotRawFileName + "." + Hashes.getRandom(4);
String returnValue = suspendDevice(dmDeviceName);
if(!returnValue.contains(foundLVMVolumeInfo.getVgName().replaceAll("-", "--"))) {
db.rollback();
resumeDevice(dmDeviceName);
throw new EucalyptusCloudException("Could not suspend device " + dmDeviceName);
}
dupFile(snapshotRawFileName, dupSnapshotDeltaFileName);
returnValue = resumeDevice(dmDeviceName);
returnValues.add(dupSnapshotDeltaFileName);
}
db.commit();
} else {
db.rollback();
throw new EucalyptusCloudException();
Expand Down
Loading

0 comments on commit be15a9a

Please sign in to comment.