Windows installation failure: fastBPE(setup.py) error #1224
Comments
|
Ok. I have solved this problem myself. This error arises when you try to compile fastBPE( a dependency of fairseq) in windows. In order to solve this, you have to make modifications on:
First, git clone fastBPE source code to your local directory. Change line 15: #include <mman.c>to #include "mman.c"Add /*
* sys/mman.h
* mman-win32
*/
#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_
#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later.
#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows.
#endif
/* All the headers include this file. */
#ifndef _MSC_VER
#include <_mingw.h>
#endif
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
#define PROT_NONE 0
#define PROT_READ 1
#define PROT_WRITE 2
#define PROT_EXEC 4
#define MAP_FILE 0
#define MAP_SHARED 1
#define MAP_PRIVATE 2
#define MAP_TYPE 0xf
#define MAP_FIXED 0x10
#define MAP_ANONYMOUS 0x20
#define MAP_ANON MAP_ANONYMOUS
#define MAP_FAILED ((void *)-1)
/* Flags for msync. */
#define MS_ASYNC 1
#define MS_SYNC 2
#define MS_INVALIDATE 4
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
int munmap(void *addr, size_t len);
int mprotect(void *addr, size_t len, int prot);
int msync(void *addr, size_t len, int flags);
int mlock(const void *addr, size_t len);
int munlock(const void *addr, size_t len);
#ifdef __cplusplus
};
#endif
#endif /* _SYS_MMAN_H_ */and also #include <windows.h>
#include <errno.h>
#include <io.h>
#include "mman.h"
#ifndef FILE_MAP_EXECUTE
#define FILE_MAP_EXECUTE 0x0020
#endif /* FILE_MAP_EXECUTE */
static int __map_mman_error(const DWORD err, const int deferr)
{
if (err == 0)
return 0;
//TODO: implement
return err;
}
static DWORD __map_mmap_prot_page(const int prot)
{
DWORD protect = 0;
if (prot == PROT_NONE)
return protect;
if ((prot & PROT_EXEC) != 0)
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
}
else
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_READWRITE : PAGE_READONLY;
}
return protect;
}
static DWORD __map_mmap_prot_file(const int prot)
{
DWORD desiredAccess = 0;
if (prot == PROT_NONE)
return desiredAccess;
if ((prot & PROT_READ) != 0)
desiredAccess |= FILE_MAP_READ;
if ((prot & PROT_WRITE) != 0)
desiredAccess |= FILE_MAP_WRITE;
if ((prot & PROT_EXEC) != 0)
desiredAccess |= FILE_MAP_EXECUTE;
return desiredAccess;
}
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
{
HANDLE fm, h;
void * map = MAP_FAILED;
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4293)
#endif
const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
const DWORD protect = __map_mmap_prot_page(prot);
const DWORD desiredAccess = __map_mmap_prot_file(prot);
const off_t maxSize = off + (off_t)len;
const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
#ifdef _MSC_VER
#pragma warning(pop)
#endif
errno = 0;
if (len == 0
/* Unsupported flag combinations */
|| (flags & MAP_FIXED) != 0
/* Usupported protection combinations */
|| prot == PROT_EXEC)
{
errno = EINVAL;
return MAP_FAILED;
}
h = ((flags & MAP_ANONYMOUS) == 0) ?
(HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;
if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
{
errno = EBADF;
return MAP_FAILED;
}
fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
if (fm == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
CloseHandle(fm);
if (map == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
return map;
}
int munmap(void *addr, size_t len)
{
if (UnmapViewOfFile(addr))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mprotect(void *addr, size_t len, int prot)
{
DWORD newProtect = __map_mmap_prot_page(prot);
DWORD oldProtect = 0;
if (VirtualProtect(addr, len, newProtect, &oldProtect))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int msync(void *addr, size_t len, int flags)
{
if (FlushViewOfFile(addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mlock(const void *addr, size_t len)
{
if (VirtualLock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int munlock(const void *addr, size_t len)
{
if (VirtualUnlock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
Reason : Under any windows C++ compiler, we don't have |
|
Second:
Build modified fastBPE using: python setup.py build --compiler=mingw32And then install it: |
|
Third, two errors will occur in the last step. The fix to the following issue is similar to this page. We modify In around line 85 we change to: ### PATCH###############################
# INCLUDES NEWEST mscvcr VERSION
#########################################
elif msc_ver >= '1900':
# Visual Studio 2015 / Visual C++ 14.0
# "msvcr140.dll no longer exists" http://blogs.msdn.com/b/vcblog/archive/2014/06/03/visual-studio-14-ctp.aspx
return ['vcruntime140']
else:
raise ValueError("Unknown MS Compiler version %s " % msc_ver)Notice, my compiler version is 1915( to chekc your version, And then you need to copy a After this step, you should be able to compile successfully albeit with some warnings. And then you can install and then pip install fairseq |
|
These solutions did not work for me sadly. At the end of the third stage when I run |
|
@aliyesilkanat Looks like you are stuck at the fastBPE. Did you use that |
|
@aliyesilkanat I tried a few days, finally, I got it. This tutorial provided by @zjplab is TOTALLY ESSENTIAL to anyone persists in Windows 10 python environment |
|
where is "#include <mman.c>" ? |
|
I cannot locate the mman.c line as well |
|
|
@zjplab i followed your instructions Im getting this error 'thread' was not declared in this scope 28 | const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency()))) Full stack trace: Tired to debug and search for this issue in other portals but was not able to get any solutions could you please assist
|
|
Just adding comments of own experiences to help others with this. For reference I'm trying to get fastBPE for use for the salesforce CTRL model here: https://github.com/salesforce/ctrl Problem 1:
Problem 2:
Problem 3:
UPDATE:
UPDATE 2:
|
|
when I use "pip install fastBPE" it error. Can you help me fix error? Microsoft Windows [Version 10.0.17763.1282] C:\Users\dorae>pip install fastBPE
|
|
@ajbot2019 I followed your instruction but it was error. |
so how did you solve it finally? |
|
This is how I solved it, second to zjplab but slightly different: git clone https://github.com/glample/fastBPE.git ** BE CAREFULL from here * --->(3) notepad mman.c #include <windows.h> #include "mman.h" #ifndef FILE_MAP_EXECUTE static int __map_mman_error(const DWORD err, const int deferr) static DWORD __map_mmap_prot_page(const int prot) } static DWORD __map_mmap_prot_file(const int prot) } void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off) #ifdef _MSC_VER #ifdef _MSC_VER } int munmap(void *addr, size_t len) } int mprotect(void *addr, size_t len, int prot) } int msync(void *addr, size_t len, int flags) } int mlock(const void *addr, size_t len) } int munlock(const void *addr, size_t len) } -->(4) notepad mman.h /*
#ifndef SYS_MMAN_H #ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later. /* All the headers include this file. */ #include <sys/types.h> #ifdef __cplusplus #define PROT_NONE 0 #define MAP_FILE 0 #define MAP_FAILED ((void *)-1) /* Flags for msync. */ void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off); #ifdef __cplusplus #endif /* SYS_MMAN_H */ -->(5) go back "cd.." YES that is it ! Annoying really, a globally used package is just so fully untested ! |
Hello đồng hương, I followed your solution but still the error of unistd.h: Can you suggest any action to do please ? |
|
Hi, @calibretaliation Did you come up with any fix regarding the error of unistd.h ? |
|
I have followed how to install fastBPE in Windows by @zjplab. However, when I run "python setup.py build --compiler=mingw32", it produces an error: How can I fix it ? |
|
Copy the file fastBPE.cpython-38-x86_64-linux-gnu.so and the folder fastBPE-0.1.0.dist-info in Ubuntu to your Windows envs |
|
Hello, @liugj101. May you share the file to the public? |
I am windows 10 user. Anaconda version torch-cpu 1.1.0
I installed MSVC v142 VS2019 C++ x64/x86 building tools (v14.23)
when I tried to
pip install fairseq, I got the following errors:The text was updated successfully, but these errors were encountered: