Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

192 lines (150 sloc) 8.306 kB
libhdfs=dlopen("libhdfs")
_hdfsConnectAsUser= dlsym(libhdfs, :hdfsConnectAsUser)
_hdfsConnect= dlsym(libhdfs, :hdfsConnect)
_hdfsDisconnect= dlsym(libhdfs, :hdfsDisconnect)
_hdfsOpenFile= dlsym(libhdfs,:hdfsOpenFile)
_hdfsCloseFile= dlsym(libhdfs, :hdfsCloseFile)
_hdfsExists= dlsym(libhdfs, :hdfsExists)
_hdfsSeek= dlsym(libhdfs, :hdfsSeek)
_hdfsTell= dlsym(libhdfs, :hdfsTell)
_hdfsRead= dlsym(libhdfs, :hdfsRead)
_hdfsPread= dlsym(libhdfs, :hdfsPread)
_hdfsWrite= dlsym(libhdfs, :hdfsWrite)
_hdfsFlush= dlsym(libhdfs, :hdfsFlush)
_hdfsAvailable= dlsym(libhdfs, :hdfsAvailable)
_hdfsCopy= dlsym(libhdfs, :hdfsCopy)
_hdfsMove= dlsym(libhdfs, :hdfsMove)
_hdfsDelete= dlsym(libhdfs, :hdfsDelete)
_hdfsRename= dlsym(libhdfs, :hdfsRename)
_hdfsGetWorkingDirectory= dlsym(libhdfs, :hdfsGetWorkingDirectory)
_hdfsSetWorkingDirectory= dlsym(libhdfs, :hdfsSetWorkingDirectory)
_hdfsCreateDirectory= dlsym(libhdfs, :hdfsCreateDirectory)
_hdfsSetReplication= dlsym(libhdfs, :hdfsSetReplication)
_hdfsListDirectory=dlsym(libhdfs, :hdfsListDirectory)
_hdfsGetPathInfo=dlsym(libhdfs, :hdfsGetPathInfo)
_hdfsFreeFileInfo=dlsym(libhdfs, :hdfsFreeFileInfo)
_hdfsGetHosts=dlsym(libhdfs, :hdfsGetHosts)
_hdfsFreeHosts=dlsym(libhdfs, :hdfsFreeHosts)
_hdfsGetDefaultBlockSize= dlsym(libhdfs, :hdfsGetDefaultBlockSize)
_hdfsGetCapacity= dlsym(libhdfs, :hdfsGetCapacity)
_hdfsGetUsed= dlsym(libhdfs, :hdfsGetUsed)
_hdfsChown=dlsym(libhdfs, :hdfsChown)
_hdfsChmod=dlsym(libhdfs, :hdfsChmod)
_hdfsUtime=dlsym(libhdfs, :hdfsUtime)
## used to enforce typing ##
type HdfsFS
ptr::Ptr{Void}
end
type HdfsFile
ptr::Ptr{Void}
end
type HdfsFileInfo
ptr::Ptr{Void}
end
## libhdfs functions as ccalls ##
# strings can be passed to Ptr{Uint8} (char *)
# functions segfault if passed bad hdfsFS arguments
# multiple dispatch is used for convenience
# commented functions have not been tried out
function hdfs_connect_as_user(host, port, user)
fs = ccall(_hdfsConnectAsUser, Ptr{Void}, (Ptr{Uint8},Int32,Ptr{Uint8}),host,port,user)
return HdfsFS(fs)
end
hdfs_connect_as_user(host, port)=hdfs_connect(host, port)
hdfs_connect_as_user()=hdfs_connect()
function hdfs_connect(host,port)
fs = ccall(_hdfsConnect, Ptr{Void}, (Ptr{Uint8},Int32),host,port)
return HdfsFS(fs)
end
function hdfs_connect()
fs = ccall(_hdfsConnect, Ptr{Void}, (Ptr{Uint8},Int32),"default",0)
return HdfsFS(fs)
end
hdfs_disconnect(fs::HdfsFS)=ccall(_hdfsDisconnect, Int32, (Ptr{Void},),fs.ptr)
# file control flags need to be done manually
function hdfs_open_file(fs::HdfsFS,path,flags,bufferSize,replication,blocksize)
file = ccall(_hdfsOpenFile, Ptr{Void}, (Ptr{Void},Ptr{Uint8},Int32,Int32,Int16,Int32),fs.ptr,path,flags,bufferSize,replication,blocksize)
return HdfsFile(file)
end
function hdfs_open_file(fs::HdfsFS,path,flags)
file = ccall(_hdfsOpenFile, Ptr{Void}, (Ptr{Void},Ptr{Uint8},Int32,Int32,Int16,Int32),fs.ptr,path,flags,0,0,0)
return HdfsFile(file)
end
hdfs_close_file(fs::HdfsFS, file::HdfsFile)=ccall(_hdfsCloseFile,Int32,(Ptr{Void},Ptr{Void}),fs.ptr,file.ptr)
hdfs_exists(fs::HdfsFS, path)=ccall(_hdfsExists,Int32,(Ptr{Void},Ptr{Uint8}),fs.ptr,path)
hdfs_seek(fs::HdfsFS, file::HdfsFile, desiredPos)=ccall(_hdfsSeek,Int32,(Ptr{Void},Ptr{Void},Int64),fs.ptr, file.ptr, desiredPos)
hdfs_tell(fs::HdfsFS, file::HdfsFile)=ccall(_hdfsTell,Int64,(Ptr{Void},Ptr{Void}), fs.ptr, file.ptr)
hdfs_read(fs::HdfsFS, file::HdfsFile, buffer, length)=ccall(_hdfsRead,Int32, (Ptr{Void},Ptr{Void},Ptr{Void},Int32), fs.ptr, file.ptr, buffer, length)
function hdfs_read(fs::HdfsFS, file::HdfsFile, length)
buffer = Array(Uint8,length)
r = ccall(_hdfsRead,Int32, (Ptr{Void},Ptr{Void},Ptr{Void},Int32), fs.ptr, file.ptr, buffer, length)
if r==-1
error("-1")
else
print("read ",r," bytes\n")
return buffer
end
end
hdfs_pread(fs::HdfsFS, file::HdfsFile, position, buffer, length)=ccall(_hdfsPread,Int32, (Ptr{Void},Ptr{Void},Int64,Ptr{Void},Int32), fs.ptr, file.ptr, position, buffer, length)
function hdfs_pread(fs::HdfsFS, file::HdfsFile, position, length)
buffer = Array(Uint8,length)
r = ccall(_hdfsPread,Int32, (Ptr{Void},Ptr{Void},Ptr{Void},Int32), fs.ptr, file.ptr, buffer, position, length)
if r==-1
error("-1")
else
print("read ",r," bytes\n")
return buffer
end
end
#can be passed an ASCIIString (length not necessary in that case)
hdfs_write(fs::HdfsFS, file::HdfsFile, buffer, length)=ccall(_hdfsWrite,Int32,(Ptr{Void},Ptr{Void},Ptr{Void},Int32),fs.ptr, file.ptr, buffer, length)
hdfs_write(fs::HdfsFS, file::HdfsFile, buffer::ASCIIString, length)=ccall(_hdfsWrite,Int32,(Ptr{Void},Ptr{Void},Ptr{Void},Int32),fs.ptr, file.ptr, convert(Ptr{Uint8},buffer), length)
hdfs_write(fs::HdfsFS, file::HdfsFile, buffer::ASCIIString)=ccall(_hdfsWrite,Int32,(Ptr{Void},Ptr{Void},Ptr{Void},Int32),fs.ptr, file.ptr, convert(Ptr{Uint8},buffer), length(buffer))
hdfs_flush(fs::HdfsFS, file::HdfsFile)=ccall(_hdfsFlush, Int32, (Ptr{Void},Ptr{Void}), fs.ptr, file.ptr)
hdfs_available(fs::HdfsFS, file::HdfsFile)=ccall(_hdfsAvailable, Int32, (Ptr{Void},Ptr{Void}), fs.ptr, file.ptr)
hdfs_copy(srcFS::HdfsFS, src, dstFS::HdfsFS, dst)=ccall(_hdfsCopy, Int32, (Ptr{Void},Ptr{Uint8},Ptr{Void},Ptr{Uint8}), srcFS.ptr, src, dstFS.ptr, dst)
hdfs_move(srcFS::HdfsFS, src, dstFS::HdfsFS, dst)=ccall(_hdfsMove, Int32, (Ptr{Void},Ptr{Uint8},Ptr{Void},Ptr{Uint8}), srcFS.ptr, src, dstFS.ptr, dst)
hdfs_delete(fs::HdfsFS, path)=ccall(_hdfsDelete, Int32, (Ptr{Void},Ptr{Uint8}), fs.ptr, path)
hdfs_rename(fs::HdfsFS, oldPath, newPath)=ccall(_hdfsRename, Int32, (Ptr{Void}, Ptr{Uint8},Ptr{Uint8}), fs.ptr, oldPath, newPath)
hdfs_get_working_directory(fs::HdfsFS, buffer, bufferSize)=ccall(_hdfsGetWorkingDirectory, Ptr{Uint8}, (Ptr{Void}, Ptr{Uint8}, Int32), fs.ptr, buffer, bufferSize)
function hdfs_get_working_directory(fs::HdfsFS,bufferSize)
buffer = Array(Uint8, bufferSize)
path = ccall(_hdfsGetWorkingDirectory, Ptr{Uint8}, (Ptr{Void}, Ptr{Uint8}, Int32), fs.ptr, buffer, bufferSize)
return cstring(path)
end
function hdfs_get_working_directory(fs::HdfsFS)
buffer = Array(Uint8, 128)
path = ccall(_hdfsGetWorkingDirectory, Ptr{Uint8}, (Ptr{Void}, Ptr{Uint8}, Int32), fs.ptr, buffer, 128)
return cstring(path)
end
hdfs_set_working_directory(fs::HdfsFS, path)=ccall(_hdfsSetWorkingDirectory, Int32, (Ptr{Void}, Ptr{Uint8}), fs.ptr, path)
hdfs_create_directory(fs::HdfsFS, path)=ccall(_hdfsCreateDirectory, Int32, (Ptr{Void}, Ptr{Uint8}), fs.ptr, path)
hdfs_set_replication(fs::HdfsFS, path, replication)=ccall(_hdfsSetReplication, Int32, (Ptr{Void}, Ptr{Uint8}, Int16), fs.ptr, path, replication)
#function hdfs_list_directory(fs::HdfsFS, path, numEntries)
# fileInfo = ccall(_hdfsListDirectory, Ptr{Void}, (Ptr{Void}, Ptr{Uint8}, Int32), fs.ptr, path, numEntries)
# return HdfsFileInfo(fileInfo)
#end
#function hdfs_get_path_info(fs::HdfsFS, path)
# fileInfo = ccall(_hdfsGetPathInfo, Ptr{Void}, (Ptr{Void}, Ptr{Uint8}), fs.ptr, path)
# return HdfsFileInfo(fileInfo)
#end
#hdfs_free_file_info(fileInfo::HdfsFileInfo, numEntries)=ccall(_hdfsFreeFileInfo, Void, (Ptr{Void}, Int32), fileInfo.ptr, numEntries)
#hdfs_get_hosts(fs::HdfsFS, path, start, length)=ccall(_hdfsGetHosts, Ptr{Ptr{Ptr{Uint8}}}, (Ptr{Void}, Ptr{Uint8}, Int64, Int64), fs.ptr, path, start, length)
#hdfs_free_hosts(blockHosts)=ccall(_hdfsFreeHosts, Void, (Ptr{Ptr{Ptr{Uint8}}},), blockHosts)
hdfs_get_default_block_size(fs::HdfsFS)=ccall(_hdfsGetDefaultBlockSize, Int64, (Ptr{Void},),fs.ptr)
hdfs_get_capacity(fs::HdfsFS)=ccall(_hdfsGetCapacity, Int64, (Ptr{Void},),fs.ptr)
hdfs_get_used(fs::HdfsFS)=ccall(_hdfsGetUsed, Int64, (Ptr{Void},),fs.ptr)
#hdfs_chown(fs::HdfsFS, path, owner, group)=ccall(_hdfsChown, Int32, (Ptr{Void}, Ptr{Uint8}, Ptr{Uint8}, Ptr{Uint8}), fs.ptr, path, owner, group)
#hdfs_chmod(fs::HdfsFS, path, mode)=ccall(_hdfsChmod, Int32, (Ptr{Void}, Ptr{Uint8}, Int16), fs.ptr, path, mode)
#last two arguments are to be time_t (system dependent)
#hdfs_utime(fs::HdfsFS, path, mtime, atime)=ccall(_hdfsUtime, Int32, (Ptr{Void}, Ptr{Uint8}, int, int), fs.ptr, path, mtime, atime)
## file control flags ##
# these are system-dependent
#Int32 O_WRONLY=1
#Int32 O_RDONLY=0
#Int32 O_RDWR=2
#Int32 O_CREAT=64
#Int32 O_EXCL=128
#Int32 O_APPEND=1024
#Int32 O_TRUNC=512
#Int32 O_NONBLOCK=2048
Jump to Line
Something went wrong with that request. Please try again.