add support for alternative cmaes implementation

mypy.ini

@@ -1,6 +1,6 @@
 [mypy]
 
-[mypy-scipy.*,pandas,gym,matplotlib.*,pytest,cma,bayes_opt.*,torch.*,mpl_toolkits.*]
+[mypy-scipy.*,pandas,gym,matplotlib.*,pytest,cma,bayes_opt.*,torch.*,mpl_toolkits.*,fcmaes.*]
 ignore_missing_imports=True
 
 [mypy-nevergrad.functions.rl.agents,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc]

nevergrad/optimization/experimentalvariants.py

@@ -39,6 +39,8 @@
 # CMA
 MilliCMA = ParametrizedCMA(scale=1e-3).set_name("MilliCMA", register=True)
 MicroCMA = ParametrizedCMA(scale=1e-6).set_name("MicroCMA", register=True)
+FCMAs03 = ParametrizedCMA(fcmaes=True, scale=0.3).set_name("FCMAs03", register=True)
+FCMAp13 = ParametrizedCMA(fcmaes=True, scale=0.1, popsize=13).set_name("FCMAp13", register=True)
 
 # OnePlusOne
 FastGADiscreteOnePlusOne = ParametrizedOnePlusOne(mutation="fastga").set_name(

nevergrad/optimization/optimizerlib.py

@@ -188,23 +188,36 @@ def __init__(
             budget: Optional[int] = None,
             num_workers: int = 1,
             scale: float = 1.0,
-            diagonal: bool = False
+            popsize: Optional[int] = None,
+            diagonal: bool = False,
+            fcmaes: bool = False
     ) -> None:
         super().__init__(parametrization, budget=budget, num_workers=num_workers)
         self._scale = scale
+        self._popsize = popsize
         self._diagonal = diagonal
+        self._fcmaes = fcmaes
         self._es: Optional[cma.CMAEvolutionStrategy] = None
         # delay initialization to ease implementation of variants
         self.listx: tp.List[ArrayLike] = []
         self.listy: tp.List[float] = []
         self.to_be_asked: tp.Deque[np.ndarray] = deque()
 
     @property
-    def es(self) -> cma.CMAEvolutionStrategy:
+    def es(self) -> tp.Any:  # typing not possible since cmaes not imported :(
         if self._es is None:
-            popsize = max(self.num_workers, 4 + int(3 * np.log(self.dimension)))
-            inopts = {"popsize": popsize, "randn": self._rng.randn, "CMA_diagonal": self._diagonal, "verbose": 0}
-            self._es = cma.CMAEvolutionStrategy(x0=np.zeros(self.dimension, dtype=np.float), sigma0=self._scale, inopts=inopts)
+            popsize = max(self.num_workers, 4 + int(3 * np.log(self.dimension))) if self._popsize is None else self._popsize
+            if self._fcmaes:
+                try:
+                    from fcmaes import cmaes
+                except ImportError as e:
+                    raise ImportError("Please install fcmaes (pip install fcmaes) to use FCMA optimizers") from e
+                self._es = cmaes.Cmaes(x0=np.zeros(self.dimension, dtype=np.float),
+                                   input_sigma=self._scale,
+                                   popsize=popsize, randn=self._rng.randn)
+            else:
+                inopts = {"popsize": popsize, "randn": self._rng.randn, "CMA_diagonal": self._diagonal, "verbose": 0}
+                self._es = cma.CMAEvolutionStrategy(x0=np.zeros(self.dimension, dtype=np.float), sigma0=self._scale, inopts=inopts)
         return self._es
 
     def _internal_ask(self) -> ArrayLike:
@@ -217,7 +230,10 @@ def _internal_tell(self, x: ArrayLike, value: float) -> None:
         self.listy += [value]
         if len(self.listx) >= self.es.popsize:
             try:
-                self.es.tell(self.listx, self.listy)
+                if self._fcmaes:
+                    self.es.tell(self.listy, self.listx)
+                else:
+                    self.es.tell(self.listx, self.listy)
             except RuntimeError:
                 pass
             else:
@@ -227,10 +243,10 @@ def _internal_tell(self, x: ArrayLike, value: float) -> None:
     def _internal_provide_recommendation(self) -> ArrayLike:
         if self._es is None:
             raise RuntimeError("Either ask or tell method should have been called before")
-        if self.es.result.xbest is None:
+        cma_best = self.es.best_x if self._fcmaes else self.es.result.xbest
+        if cma_best is None:
             return self.current_bests["pessimistic"].x
-        return self.es.result.xbest  # type: ignore
-
+        return cma_best
 
 class ParametrizedCMA(base.ConfiguredOptimizer):
     """CMA-ES optimizer, wrapping external implementation: https://github.com/CMA-ES/pycma
@@ -239,22 +255,35 @@ class ParametrizedCMA(base.ConfiguredOptimizer):
     ----------
     scale: float
         scale of the search
+    popsize: Optional[int] = None
+        population size, should be n * self.num_workers for int n >= 1.
+        default is max(self.num_workers, 4 + int(3 * np.log(self.dimension)))
+
     diagonal: bool
         use the diagonal version of CMA (advised in big dimension)
+    fcmaes: bool = False
+        use fast implementation, doesn't support diagonal=True. 
+        produces equivalent results, preferable for high dimensions or
+        if objective function evaluation is fast. 
     """
 
     # pylint: disable=unused-argument
     def __init__(
         self,
         *,
         scale: float = 1.0,
-        diagonal: bool = False
+        popsize: Optional[int] = None,
+        diagonal: bool = False,
+        fcmaes: bool = False
     ) -> None:
-        super().__init__(_CMA, locals())
-
+        super().__init__(_CMA, locals())    
+        if fcmaes:
+            if diagonal:
+                raise RuntimeError("fcmaes doesn't support diagonal=True, use fcmaes=False")
 
 CMA = ParametrizedCMA().set_name("CMA", register=True)
 DiagonalCMA = ParametrizedCMA(diagonal=True).set_name("DiagonalCMA", register=True)
+FCMA = ParametrizedCMA(fcmaes=True).set_name("FCMA", register=True)
 
 
 class _PopulationSizeController:

nevergrad/optimization/recorded_recommendations.csv

@@ -50,6 +50,9 @@ DoubleFastGADiscreteOnePlusOne,0.0,0.0,0.0,0.0,,,,,,,,,,,,
 DoubleFastGAOptimisticNoisyDiscreteOnePlusOne,0.0,0.0,0.0,0.0,,,,,,,,,,,,
 EDA,-0.0691450987,-0.3901349698,-0.195989244,1.4401961148,0.8133730167,0.4021844027,-0.9366618858,-0.9048970955,-0.493399994,-0.0074111222,,,,,,
 ES,1.1400386808,0.3380024444,0.4755144618,2.6390460807,0.6911075733,1.111235567,-0.2576843178,-1.1959512855,,,,,,,,
+FCMA,1.012515477,-0.9138691467,-1.0295302074,1.2097964496,,,,,,,,,,,,
+FCMAp13,0.1012515477,-0.0913869147,-0.1029530207,0.120979645,,,,,,,,,,,,
+FCMAs03,0.3037546431,-0.274160744,-0.3088590622,0.3629389349,,,,,,,,,,,,
 FastGADiscreteOnePlusOne,0.7531428339,0.0,0.0,1.095956118,,,,,,,,,,,,
 FastGANoisyDiscreteOnePlusOne,-1.2151688011,0.0,0.0,1.095956118,,,,,,,,,,,,
 FastGAOptimisticNoisyDiscreteOnePlusOne,0.7531428339,0.0,0.0,1.095956118,,,,,,,,,,,,

requirements/bench.txt

@@ -5,3 +5,4 @@ matplotlib>=2.2.3
 gym>=0.12.1
 torch>=1.2.0
 hiplot
+fcmaes>=0.9.5.6