-
Notifications
You must be signed in to change notification settings - Fork 0
/
namenode.py
62 lines (56 loc) · 2.38 KB
/
namenode.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
import os, json, hashlib
from math import pow, ceil
from collections import OrderedDict
class Namenode:
root = "/var/dfs_nm"
blocksize = 15 * int(pow(2, 10)) #15 KB
datanodes ={('localhost',5000):5, ('localhost',6000):0, ('localhost',7000):1, ('localhost',8000):3}
#Initializes the namenode and creates nm directory if not exist
def __init__(self):
if not (os.path.isdir(self.root) and os.path.exists(self.root)):
try:
os.makedirs(self.root)
except OSError as exception:
print "Run the script as root"
#choose the datanodes with least number of blocks
def getNewDN(self, numBlocks):
dnodes = []
od = OrderedDict(sorted(self.datanodes.items()), key=lambda t: t[1])
keys = od.keys()[:-1]
if (numBlocks > len(self.datanodes)):
for dn in range(numBlocks):
dnodes.append(keys[dn%len(keys)])
else:
dnodes = keys[:numBlocks]
self.updateDN(dnodes)
return dnodes
#updates the datanodes with numbers of blocks
def updateDN(self, dnodes):
for node in dnodes:
self.datanodes[node] += 1
#filename => array of blocks
def get(self, file_path, timestamp):
pathHash = hashlib.sha1(file_path).hexdigest()
filename = pathHash + "@" + str(timestamp)
dirPath = os.path.join(self.root, os.path.dirname(file_path))
if not (os.path.isdir(dirPath) and os.path.exists(dirPath)):
raise Exception('{} doesnt exist'.format(dirPath))
absPath = os.path.join(dirPath, filename)
arr = None
with open(absPath, 'r') as f:
list_datanodes = f.read()
print json.loads(list_datanodes)
return list_datanodes
#filename, filesize =>
def save(self, file_path, file_size, timestamp):
numBlocks = int(ceil(float(file_size)/self.blocksize))
pathHash = hashlib.sha1(file_path).hexdigest()
filename = pathHash + "@" + str(timestamp)
dirPath = os.path.join(self.root, os.path.dirname(file_path))
if not (os.path.isdir(dirPath) and os.path.exists(dirPath)):
os.makedirs(dirPath)
absPath = os.path.join(dirPath, filename)
with open(absPath, 'w') as f:
list_ports = json.dumps(self.getNewDN(numBlocks))
f.write(list_ports)
return list_ports