Unable to embed python interpreter on *nix for python >3.7.3

[nehaljwani]

On Linux:

(/spare/scratch/1573348376/dev2) [nwani@coredump 1573348376 ]$ cc a.c -I$CONDA_PREFIX/include/python3.7m -L$CONDA_PREFIX/lib -lpython3.7m -Wl,-rpath,$CONDA_PREFIX/lib
(/spare/scratch/1573348376/dev2) [nwani@coredump 1573348376 ]$ ./a.out
Could not find platform independent libraries <prefix>
Could not find platform dependent libraries <exec_prefix>
Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]
Fatal Python error: initfsencoding: Unable to get the locale encoding
ModuleNotFoundError: No module named 'encodings'

Current thread 0x00007f0695889700 (most recent call first):
Aborted

On macOS:

(/private/tmp/scratch.nwani/1573355459/dev) /t/s/1573355459 ❯❯❯ cc a.c -I$CONDA_PREFIX/include/python3.8 -L$CONDA_PREFIX/lib -lpython3.8 -Wl,-rpath,$CONDA_PREFIX/lib                                ✘ 134
(/private/tmp/scratch.nwani/1573355459/dev) /t/s/1573355459 ❯❯❯ ./a.out
Could not find platform independent libraries <prefix>
Could not find platform dependent libraries <exec_prefix>
Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]
Python path configuration:
  PYTHONHOME = (not set)
  PYTHONPATH = (not set)
  program name = './a.out'
  isolated = 0
  environment = 1
  user site = 1
  import site = 1
  sys._base_executable = '/private/tmp/scratch.nwani/1573355459/a.out'
  sys.base_prefix = '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol'
  sys.base_exec_prefix = '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol'
  sys.executable = '/private/tmp/scratch.nwani/1573355459/a.out'
  sys.prefix = '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol'
  sys.exec_prefix = '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol'
  sys.path = [
    '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol/lib/python38.zip',
    '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol/lib/python3.8',
    '/opt/concourse/worker/volumes/live/903ea60a-0489-485f-6f93-162648469061/volume/python_1573076474316/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehol/lib/lib-dynload',
  ]
Fatal Python error: init_fs_encoding: failed to get the Python codec of the filesystem encoding
Python runtime state: core initialized
ModuleNotFoundError: No module named 'encodings'

Program that I am trying to compile (copied unmodified from python.org):

#define PY_SSIZE_T_CLEAN
#include <Python.h>

int
main(int argc, char *argv[])
{
    wchar_t *program = Py_DecodeLocale(argv[0], NULL);
    if (program == NULL) {
        fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
        exit(1);
    }
    Py_SetProgramName(program);  /* optional but recommended */
    Py_Initialize();
    PyRun_SimpleString("from time import time,ctime\n"
                       "print('Today is', ctime(time()))\n");
    if (Py_FinalizeEx() < 0) {
        exit(120);
    }
    PyMem_RawFree(program);
    return 0;
}

Test this on a system with no existing systemwide installation of python3 at standard locations, e.g: vanilla CentOS6

[nehaljwani]

I have tried compiling directly from source (tag: v3.7.5) and that installation doesn't face this problem.

[nehaljwani]

If I move Py_Initialize() before Py_DecodeLocale(), then it works, but according to the documentation, it is not recommended.

[nehaljwani]

I found that the conda forge builds didn't suffer from this issue. Turns out it is a single line in the recipe which is creating this problem. Looks like the prefix detection in binary files was turned off by accident:
AnacondaRecipes/python-feedstock@54df219#diff-e178b687b10a71a3348107ae3154e44cR75

[Ryandry1st]

Has this been fixed yet? I am running into the python embed problem shown in some of the linked issues using the current anaconda distribution.

[davidhewitt]

It looks like this might affect Windows too. (PyO3/pyo3#1554)

I see that the line linked in #11439 (comment) has changed for the 3.9 branch of the feedstock repository, however testing with a 3.9 virtualenv I experience the same problem.

[cgranade]

I'm running into the same issue on Windows as well. I've found that in some cases, setting the PYTHONHOME environment variable before using PyO3 to embed Python works, but this does not seem to be reliable:

use std::env;

use pyo3::{PyResult, Python, types::PyModule};

pub fn run_example() -> PyResult<()> {
    let python_exe = which::which("python").unwrap();
    let python_home = python_exe.parent().unwrap();
    println!("Setting PYTHONHOME = {}", python_home.display());
    env::set_var("PYTHONHOME", python_home);


    let gil = Python::acquire_gil();
    let py = gil.python();

    // Find where the exe and site-packages are.
    let sys = PyModule::import(py, "sys")?;
    let exec: String = sys.getattr("executable")?.extract()?;
    let site = PyModule::import(py, "site")?;
    let site_packages: Vec<String> = site.get("getsitepackages")?.call0()?.extract()?;

    println!("Found Python at: {}", exec);

    println!("Site packages:");
    for dir in site_packages {
        println!("- {}", dir);
    }

    Ok(())
}