OSError Too many open files error while using server dispatch

[FyzHsn]

@FyzHsn commented on Thu Dec 09 2021

When using the lattice.dispatch method to dispatch a workflow with many electrons, we get the following error:

distributed.worker - WARNING - Compute Failed
Function:  to_run
args:      ('use-nice-word', '/Users/faiyaz/Code/nimbus/doc/source/tutorials/quantum_chemistry/results/')
kwargs:    {}
Exception: "OSError(24, 'Too many open files')"

To recreate this issue:

import covalent as ct

# !pip install ase
# !pip install ase-notebook
from ase import Atoms
from ase.calculators.emt import EMT
from ase.constraints import FixAtoms
from ase.optimize import QuasiNewton
from ase.build import fcc111, add_adsorbate
from ase.io import read
from ase.io.trajectory import Trajectory
from ase_notebook import AseView, ViewConfig

@ct.electron
def construct_cu_slab(
    unit_cell=(4, 4, 2),
    vacuum=10.0,
):
    slab = fcc111("Cu", size=unit_cell, vacuum=vacuum)
    return slab


@ct.electron
def compute_system_energy(system):
    system.calc = EMT()
    return system.get_potential_energy()


@ct.electron
def construct_n_molecule(d=0):
    return Atoms("2N", positions=[(0.0, 0.0, 0.0), (0.0, 0.0, d)])


@ct.electron
def get_relaxed_slab(slab, molecule, height=1.85):
    slab.calc = EMT()
    add_adsorbate(slab, molecule, height, "ontop")
    constraint = FixAtoms(mask=[a.symbol != "N" for a in slab])
    slab.set_constraint(constraint)
    dyn = QuasiNewton(slab, trajectory="/tmp/N2Cu.traj", logfile="/tmp/temp")
    dyn.run(fmax=0.01)
    return slab

@ct.lattice
def compute_energy(initial_height=3, distance=1.10):
    N2 = construct_n_molecule(d=distance)
    e_N2 = compute_system_energy(system=N2)

    slab = construct_cu_slab(unit_cell=(4, 4, 2), vacuum=10.0)
    e_slab = compute_system_energy(system=slab)

    relaxed_slab = get_relaxed_slab(slab=slab, molecule=N2, height=initial_height)
    e_relaxed_slab = compute_system_energy(system=relaxed_slab)
    final_result = e_slab + e_N2 - e_relaxed_slab

    return final_result

optimize_height = ct.electron(compute_energy)

import numpy as np


@ct.lattice
def vary_distance(distance):
    result = []
    for i in distance:
        result.append(optimize_height(initial_height=3, distance=i))
    return result

distance = np.linspace(1, 1.5, 30)
dispatch_id = vary_distance.dispatch(distance=distance)

result = ct.get_result(dispatch_id, wait=True)
print(result.status)

The error msg can be found in ~/.cache/covalent/dispatcher.log

This issue is possibly related to issue #184.

Actionables:

1. If possible, write a test that fails before implementing the bug fix.
2. Fix the Quantum Chemistry tutorial and change the serverless dispatch to the server dispatch when the bug is removed.