Skip to content
Permalink
Browse files

fix python code so it runs on windows. (#77)

  • Loading branch information...
lovettchris authored and harsha-simhadri committed Mar 16, 2019
1 parent f2203b1 commit 2aca846141d1051004cbca233639dfb51aa9e29b
@@ -287,3 +287,6 @@ __pycache__/
*.btm.cs
*.odx.cs
*.xsd.cs
.vscode/
usps10/
.ipynb_checkpoints/
@@ -234,47 +234,47 @@ def saveParams(self, currDir):
Function to save Parameter matrices
'''
if self.numMatrices[0] == 1:
np.save(currDir + '/W.npy', self.FastParams[0].eval())
np.save(os.path.join(currDir, "W.npy"), self.FastParams[0].eval())
else:
np.save(currDir + '/W1.npy', self.FastParams[0].eval())
np.save(currDir + '/W2.npy', self.FastParams[1].eval())
np.save(os.path.join(currDir, "W1.npy"), self.FastParams[0].eval())
np.save(os.path.join(currDir, "W2.npy"), self.FastParams[1].eval())

if self.numMatrices[1] == 1:
np.save(currDir + '/U.npy',
np.save(os.path.join(currDir, "U.npy"),
self.FastParams[self.numMatrices[0]].eval())
else:
np.save(currDir + '/U1.npy',
np.save(os.path.join(currDir, "U1.npy"),
self.FastParams[self.numMatrices[0]].eval())
np.save(currDir + '/U2.npy',
np.save(os.path.join(currDir, "U2.npy"),
self.FastParams[self.numMatrices[0] + 1].eval())

if self.FastObj.cellType == "FastGRNN":
np.save(currDir + '/Bg.npy',
np.save(os.path.join(currDir, "Bg.npy"),
self.FastParams[self.totalMatrices].eval())
np.save(currDir + '/Bh.npy',
np.save(os.path.join(currDir, "Bh.npy"),
self.FastParams[self.totalMatrices + 1].eval())
np.save(currDir + '/zeta.npy',
np.save(os.path.join(currDir, "zeta.npy"),
self.FastParams[self.totalMatrices + 2].eval())
np.save(currDir + '/nu.npy',
np.save(os.path.join(currDir, "nu.npy"),
self.FastParams[self.totalMatrices + 3].eval())
elif self.FastObj.cellType == "FastRNN":
np.save(currDir + '/B.npy',
np.save(os.path.join(currDir, "B.npy"),
self.FastParams[self.totalMatrices].eval())
np.save(currDir + '/alpha.npy', self.FastParams[
np.save(os.path.join(currDir, "alpha.npy"), self.FastParams[
self.totalMatrices + 1].eval())
np.save(currDir + '/beta.npy',
np.save(os.path.join(currDir, "beta.npy"),
self.FastParams[self.totalMatrices + 2].eval())
np.save(currDir + '/FC.npy', self.FC.eval())
np.save(currDir + '/FCbias.npy', self.FCbias.eval())
np.save(os.path.join(currDir, "FC.npy"), self.FC.eval())
np.save(os.path.join(currDir, "FCbias.npy"), self.FCbias.eval())

def train(self, batchSize, totalEpochs, sess,
Xtrain, Xtest, Ytrain, Ytest,
decayStep, decayRate, dataDir, currDir):
'''
The Dense - IHT - Sparse Retrain Routine for FastCell Training
'''
resultFile = open(
dataDir + '/' + str(self.FastObj.cellType) + 'Results.txt', 'a+')
fileName = str(self.FastObj.cellType) + 'Results.txt'
resultFile = open(os.path.join(dataDir, fileName), 'a+')
numIters = int(np.ceil(float(Xtrain.shape[0]) / float(batchSize)))
totalBatches = numIters * totalEpochs

@@ -386,6 +386,13 @@ def train(self, batchSize, totalEpochs, sess,

print("The Model Directory: " + currDir + "\n")

# output the tensorflow model
model_dir = os.path.join(currDir, "model")
os.makedirs(model_dir, exist_ok=True)

saver = tf.train.Saver(tf.global_variables())
saver.save(sess, os.path.join(model_dir, "model.ckpt"), global_step=totalEpochs)

resultFile.close()
self.outFile.flush()
if self.outFile is not sys.stdout:
@@ -3,43 +3,45 @@
#
# Setting up the USPS Data.

import subprocess
import bz2
import os
import subprocess
import sys

import requests
import numpy as np
from sklearn.datasets import load_svmlight_file
import sys
from helpermethods import download_file, decompress

def downloadData(workingDir, downloadDir, linkTrain, linkTest):
def runcommand(command):
p = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
output, error = p.communicate()
assert(p.returncode == 0), 'Command failed: %s' % command


def downloadData(workingDir, downloadDir, linkTrain, linkTest):
path = workingDir + '/' + downloadDir
path = os.path.abspath(path)
try:
os.mkdir(path)
os.makedirs(path, exist_ok=True)
except OSError:
print("Could not create %s. Make sure the path does" % path)
print("not already exist and you have permisions to create it.")
print("not already exist and you have permissions to create it.")
return False
cwd = os.getcwd()
os.chdir(path)
print("Downloading data")
command = 'wget %s' % linkTrain
runcommand(command)
command = 'wget %s' % linkTest
runcommand(command)
print("Extracting data")
command = 'bzip2 -d usps.bz2'
runcommand(command)
command = 'bzip2 -d usps.t.bz2'
runcommand(command)
command = 'mv usps train.txt'
runcommand(command)
command = 'mv usps.t test.txt'
runcommand(command)
os.chdir(cwd)

training_data_bz2 = download_file(linkTrain, path)
test_data_bz2 = download_file(linkTest, path)

training_data = decompress(training_data_bz2)
test_data = decompress(test_data_bz2)

train = os.path.join(path, "train.txt")
test = os.path.join(path, "test.txt")
if os.path.isfile(train):
os.remove(train)
if os.path.isfile(test):
os.remove(test)

os.rename(training_data, train)
os.rename(test_data, test)
os.remove(training_data_bz2)
os.remove(test_data_bz2)
return True

if __name__ == '__main__':
@@ -61,4 +63,4 @@ def runcommand(command):

if not downloadData(workingDir, downloadDir, linkTrain, linkTest):
exit(failureMsg)
print("Done")
print("Done: see ", downloadDir)
@@ -6,9 +6,42 @@
for the example script.
'''
import argparse
import bz2
import datetime
import os

import numpy as np
import requests


def decompress(filepath):
print("extracting: ", filepath)
zipfile = bz2.BZ2File(filepath) # open the file
data = zipfile.read() # get the decompressed data
newfilepath = os.path.splitext(filepath)[0] # assuming the filepath ends with .bz2
with open(newfilepath, 'wb') as f:
f.write(data) # write a uncompressed file
return newfilepath


def download_file(url, local_folder=None):
"""Downloads file pointed to by `url`.
If `local_folder` is not supplied, downloads to the current folder.
"""
filename = os.path.basename(url)
if local_folder:
filename = os.path.join(local_folder, filename)

# Download the file
print("Downloading: " + url)
response = requests.get(url, stream=True)
if response.status_code != 200:
raise Exception("download file failed with status code: %d, fetching url '%s'" % (response.status_code, url))

# Write the file to disk
with open(filename, "wb") as handle:
handle.write(response.content)
return filename


def checkIntPos(value):
@@ -32,29 +32,29 @@ def quantizeFastModels(modelDir, maxValue=127, scalarScaleFactor=1000):
if file.endswith("npy"):
if file.startswith("W"):
paramNameList.append(file)
temp = np.load(modelDir + "/" + file)
temp = np.load(os.path.join(modelDir, file))
paramWeightList.append(temp)
paramLimitList.append(min_max(temp, file))
elif file.startswith("U"):
paramNameList.append(file)
temp = np.load(modelDir + "/" + file)
temp = np.load(os.path.join(modelDir, file))
paramWeightList.append(temp)
paramLimitList.append(min_max(temp, file))
elif file.startswith("B"):
paramNameList.append(file)
temp = np.load(modelDir + "/" + file)
temp = np.load(os.path.join(modelDir, file))
paramWeightList.append(temp)
paramLimitList.append(min_max(temp, file))
elif file.startswith("FC"):
classifierNameList.append(file)
temp = np.load(modelDir + "/" + file)
temp = np.load(os.path.join(modelDir, file))
classifierWeightList.append(temp)
classifierLimitList.append(min_max(temp, file))
elif file.startswith("mean") or file.startswith("std"):
continue
else:
scalarNameList.append(file)
scalarWeightList.append(np.load(modelDir + "/" + file))
scalarWeightList.append(np.load(os.path.join(modelDir, file)))

paramLimit = np.max(paramLimitList)
classifierLimit = np.max(classifierLimitList)
@@ -97,30 +97,30 @@ def quantizeFastModels(modelDir, maxValue=127, scalarScaleFactor=1000):
quantScalarWeights.append(
np.round(scalarScaleFactor * sigmoid(scalar)).astype('int32'))

if os.path.isdir(modelDir + '/QuantizedFastModel') is False:
quantModelDir = os.path.join(modelDir, 'QuantizedFastModel')
if not os.path.isdir(quantModelDir):
try:
os.mkdir(modelDir + '/QuantizedFastModel')
quantModelDir = modelDir + '/QuantizedFastModel'
os.makedirs(quantModelDir, exist_ok=True)
except OSError:
print("Creation of the directory %s failed" %
modelDir + '/QuantizedFastModel')
print("Creation of the directory %s failed" % quantModelDir)

np.save(quantModelDir + "/paramScaleFactor.npy",
np.save(os.path.join(quantModelDir, "paramScaleFactor.npy"),
paramScaleFactor.astype('int32'))
np.save(quantModelDir + "/classifierScaleFactor.npy",
np.save(os.path.join(quantModelDir, "classifierScaleFactor.npy"),
classifierScaleFactor)
np.save(quantModelDir + "/scalarScaleFactor", scalarScaleFactor)
np.save(os.path.join(quantModelDir, "scalarScaleFactor"), scalarScaleFactor)

for i in range(0, len(scalarNameList)):
np.save(quantModelDir + "/q" +
scalarNameList[i], quantScalarWeights[i])
np.save(os.path.join(quantModelDir, "q" +
scalarNameList[i]), quantScalarWeights[i])

for i in range(len(classifierNameList)):
np.save(quantModelDir + "/q" +
classifierNameList[i], quantClassifierWeights[i])
np.save(os.path.join(quantModelDir, "q" +
classifierNameList[i]), quantClassifierWeights[i])

for i in range(len(paramNameList)):
np.save(quantModelDir + "/q" + paramNameList[i], quantParamWeights[i])
np.save(os.path.join(quantModelDir, "q" + paramNameList[i]),
quantParamWeights[i])

print("\n\nQuantized Model Dir: " + quantModelDir)

@@ -4,3 +4,4 @@ pandas==0.23.4
scikit-learn==0.19.2
scipy==1.1.0
tensorflow==1.10.1
requests
@@ -4,3 +4,4 @@ pandas==0.23.4
scikit-learn==0.19.2
scipy==1.1.0
tensorflow-gpu==1.10.1
requests

0 comments on commit 2aca846

Please sign in to comment.
You can’t perform that action at this time.