Update download/upload scripts for GCP.

scripts/download_incr_DB.py

@@ -57,10 +57,7 @@
 DOWNLOAD_STARTED_LIST = []
 
 def getURL():
-	if AWS_ENDPOINT_URL:
-		return f"{AWS_ENDPOINT_URL}/{BUCKET_NAME}"
-	else:
-		return "http://"+BUCKET_NAME+".s3.amazonaws.com"
+	return "http://"+BUCKET_NAME+".storage.googleapis.com"
 
 def UploadLock():
 	response = requests.get(getURL()+"/"+PERSISTENCE_SNAPSHOT_NAME+"/"+TESTNET_NAME+"/.lock")
@@ -157,7 +154,7 @@ def GetAllObjectsFromS3(url, folderName=""):
 	# Try get the entire persistence keys.
 	# S3 limitation to get only max 1000 keys. so work around using marker.
 	while True:
-		response = requests.get(url, params={"prefix":prefix, "max-keys":1000, "marker": MARKER})
+		response = requests.get(url, params={"prefix":prefix, "list-type": 1, "max-keys":1000, "marker": MARKER})
 		tree = ET.fromstring(response.text)
 		print("[" + str(datetime.datetime.now()) + "] Files to be downloaded:")
 		lastkey = ''

scripts/download_static_DB.py

@@ -45,10 +45,7 @@
 AWS_ENDPOINT_URL=os.getenv("AWS_ENDPOINT_URL")
 
 def getURL():
-	if AWS_ENDPOINT_URL:
-		return f"{AWS_ENDPOINT_URL}/{BUCKET_NAME}"
-	else:
-		return "http://"+BUCKET_NAME+".s3.amazonaws.com"
+	return "http://"+BUCKET_NAME+".storage.googleapis.com"
 
 def Diff(list1, list2):
 	return (list(list(set(list1)-set(list2)) + list(set(list2)-set(list1))))
@@ -169,25 +166,22 @@ def GetAllObjectsFromS3(url, folderName=""):
 	# Try get the entire persistence keys.
 	# S3 limitation to get only max 1000 keys. so work around using marker.
 	while True:
-		response = requests.get(url, params={"prefix":prefix, "max-keys":1000, "marker": MARKER})
+		response = requests.get(url, params={"prefix":prefix, "list-type": 1, "max-keys":1000, "marker": MARKER})
 		tree = ET.fromstring(response.text)
-		startInd = 5
-		if(tree[startInd:] == []):
-			print("Empty response")
-			return False
 		print("[" + str(datetime.datetime.now()) + "] Files to be downloaded:")
 		lastkey = ''
-		for key in tree[startInd:]:
-			key_url = key[0].text
+		for key in tree.findall("{*}Contents"):
+			# skip compressed blockchain-data file i.e. testnet-name.tar.gz
+			key_url = key.find("{*}Key").text
 			if key_url.endswith("/"):
 				continue
 			list_of_keyurls.append(url+"/"+key_url)
 			print(key_url)
 			lastkey = key_url
-		istruncated=tree[4].text
-		if istruncated == 'true':
+		is_truncated = tree.find('{*}IsTruncated').text
+		if is_truncated == 'true':
 			MARKER=lastkey
-			print(istruncated)
+			print(is_truncated)
 		else:
 			break
 

scripts/upload_incr_DB.py

@@ -55,10 +55,7 @@
 rootLogger.addHandler(std_handler)
 
 def awsS3Url():
-	if AWS_ENDPOINT_URL:
-		return f"{AWS_ENDPOINT_URL}/{BUCKET_NAME}"
-	else:
-		return "http://"+BUCKET_NAME+".s3.amazonaws.com"
+	return "http://"+BUCKET_NAME+".storage.googleapis.com"
 
 def awsCli():
     if AWS_ENDPOINT_URL: