symmetrise_lib.py

import numpy as np
import copy
import fitsio as fi
import pylab as plt
plt.switch_backend('agg')
import numpy as np
import mbii.lego_tools as utils
from numpy.core.records import fromarrays

def parse_binning(options):
    """Takes a set of configuration settings, extracts the 2pt binning info
    and converts it into a dictionary."""

    corrs = options['2pt']['ctypes'].split()
    nbins = options['2pt']['nbin']

    bindict = {}

    # If one number is given then fix all 2pt measurements to use that
    # otherwise assume the bin numbers follow the same ordering as the 
    # correlation types
    if isinstance(nbins,int):
        n0 = [nbins]*len(corrs)     
    else:
        n0 = nbins.split()

    for n,c in zip(n0,corrs):
        bindict[c] = int(n)
        print('Will use %d bins for %s'%(int(n),c))

    return bindict



def symmetrise_catalogue3(data=None, seed=4000, mask=None, filename='/home/ssamurof/massive_black_ii/subhalo_cat-nthreshold5.fits', pivot='mass', central='spatial_central', snapshot=85, savedir=None, replace=False, verbose=True, nmin=0, rank=0, size=1):
    """Takes a catalogue of subhalos with orientations, positions and host halo identifiers.
	   Cycles through the objects and applies a random rotation about the halo centre to each,
	   recalculating the position and shape vectors such that the relative orientation to
	   the centre is unchanged.
	   Then projects the 3D orientations along one axis of the simulation box and recomputes
	   the ellipticities.
	   Writes the result to a new FITS catalogue, identical to the input save for the shapes
	   and positions.
	"""

	# Set the random seed once at the top of the loop
    np.random.seed(seed)

    # Null mask if none is specified
    if mask is None:
    	mask = np.ones(data.size).astype(bool)

    # Safety catch
    print('Will write to %s'%savedir)
    if (savedir==filename):
    	print("WARNING: target and input file paths are the same")
    	import pdb ; pdb.set_trace()

    # Pull out the identifiers associating galaxies with halos
    group_ids = data['halo_id'][mask]
    groups = np.unique(group_ids)

    # Copy over the old data
    outdat = np.zeros(len(data), dtype=data.dtype)
    for name in outdat.dtype.names:
        outdat[name] = data[name]

    outdat = utils.add_col(outdat, 'x0', np.array([0]*outdat.size), dtype=float)
    outdat = utils.add_col(outdat, 'y0', np.array([0]*outdat.size), dtype=float)
    outdat = utils.add_col(outdat, 'z0', np.array([0]*outdat.size), dtype=float)

    # Now start the outer loop, over halos. Sorry.
    i0=0
    for i, g in enumerate(groups):

    	# MPI handling
        if i%size!=rank:
            continue

        # Select the galaxies in this particular halo
        select = (group_ids==g)
        ngal = len(data[select])
        if verbose:
        	print('Halo %g contains %d object(s)'%(g,ngal))

        outdat['halo_id'][select] = group_ids[select]

        # We might want to apply some threshold, and ignore halos with fewer galaxies than some threshold 
        if (ngal<nmin):
            if verbose:
            	print('Skipping halo %d as it contains <%d galaxies'%(g,nmin))
            continue

        # Now apply the symmetrisation operation to these galaxies
        symmetrised_halo = symmetrise_halo5(data[select], verbose=True, g=g, pivot=pivot, central=central, snapshot=snapshot, simulation=simulation)

        if verbose:
            print(g, ngal)

        # Transfer the new symmetrised columns to the output array
        for name in outdat.dtype.names:
            outdat[name][select] = symmetrised_halo[name]

        # Count the number of halos actually altered by the process
        i0+=ngal

    # Finally store the result as a FITS file
    outfits = fi.FITS(savedir.replace('.fits', '%d.fits'%rank), 'rw')
    outfits.write(outdat)
    outfits.close()

    print('Done')
    return 0

def symmetrise_halo4(data, verbose=True, g=None):
    """
	Takes an array of objects associated with a particular halo,
	queries the database on coma to find the halo centroid positions
	and then applies a random rotation to each.
	The output is an array in the same format as given, but with
	new positions and shapes."""

    N = data['npart_baryon']
    n = data.size

    boxsize = choose_boxsize(simulation)

    # Work out the centroid about which to rotate  
    data, positions, (x0, y0, z0) = get_wrapped_positions(g, data, snapshot=snapshot, boxsize=boxsize) 
    cent = np.array([x0,y0,z0])

    # New array to store the output
    rot = np.zeros(data.size, dtype=data.dtype)  
    for name in data.dtype.names:
        rot[name][:n] = data[name]

    rot = utils.add_col(rot, 'x0', np.array([x0]*data.size), dtype=float)
    rot = utils.add_col(rot, 'y0', np.array([y0]*data.size), dtype=float)
    rot = utils.add_col(rot, 'z0', np.array([z0]*data.size), dtype=float)

    # Start the inner loop, over individual galaxies. Sorry.
    for i in xrange(n):

        # Cartesian galaxy position, relative to the halo centre
        pos = positions.T[i]

        # Galaxy position in spherical polar coordinates
        R = np.sqrt(sum(pos*pos))
        phi = np.arccos(pos[2]/R) * pos[0]/abs(pos[0])
        theta = np.arcsin(pos[1]/R/np.sin(phi))
        rot['rh'][i] = R

        # We can skip the rest for central objects
        if (data[central_name][i]==1):
            rot = check_wrapping(i, rot)
            if verbose:
                print('skipping object -- it is classified as a central galaxy')
            continue

        if verbose:
            print(i,)

        # Construct a 3x3 matrix to rotate galaxies by a random angle in the range [0, 2/pi]
        # about a randomly chosen axis.
        Rxyz = build_rotation_matrix()

        # Rotated positions
        rotated = np.dot(Rxyz,pos)
        rot['x'][i] = copy.deepcopy(rotated[0])+x0
        rot['y'][i] = copy.deepcopy(rotated[1])+y0
        rot['z'][i] = copy.deepcopy(rotated[2])+z0

        if verbose:
            print('New position (x, y, z) : %3.3f, %3.3f %3.3f'%(rot['x'][i],rot['y'][i],rot['z'][i]))

        # Apply the same rotation to the three orientation vectors
        a3d = np.array([data['a1'][i], data['a2'][i], data['a3'][i]])
        b3d = np.array([data['b1'][i], data['b2'][i], data['b3'][i]])
        c3d = np.array([data['c1'][i], data['c2'][i], data['c3'][i]])
        arot = np.dot(Rxyz,a3d)
        brot = np.dot(Rxyz,b3d)
        crot = np.dot(Rxyz,c3d)
        axes = {'a':arot,'b':brot,'c':crot}

        # And for the dark matter component
        a3d_dm = np.array([data['a1_dm'][i], data['a2_dm'][i], data['a3_dm'][i]])
        b3d_dm = np.array([data['b1_dm'][i], data['b2_dm'][i], data['b3_dm'][i]])
        c3d_dm = np.array([data['c1_dm'][i], data['c2_dm'][i], data['c3_dm'][i]])
        arot_dm = np.dot(Rxyz,a3d_dm)
        brot_dm = np.dot(Rxyz,b3d_dm)
        crot_dm = np.dot(Rxyz,c3d_dm)
        axes_dm = {'a':arot_dm,'b':brot_dm,'c':crot_dm}

        # Stays the same
        rot['npart_baryon'][i] = data['npart_baryon'][i]

        # We also need to shift a few objects near the edges, which the rotation leaves outside the simulation box,
        # back to the other side of the universe 
        if verbose: 
            print('Rotation leaves %d object(s) outside the simulation box.'%(rot['x'][(rot['x']<0) | (rot['y']<0) | (rot['z']<0)].size))

        rot = check_wrapping(i, rot)

        if rot['x'][i]<0 : import pdb ; pdb.set_trace()

        # Store everything in the appropriate place
        for j in xrange(3):
        	for axis in ['a','b','c']:
        		rot['%c%d'%(axis,j+1)][i] =  axes[axis][j]
        		rot['%c%d_dm'%(axis,j+1)][i] =  axes_dm[axis][j]

        # Work out the projected 2D ellipticities for the stellar component
        rot = project_ellipticities(i, rot, suffix='')
        # Same for dark matter 
        rot = project_ellipticities(i, rot, suffix='dm')
        
    return rot


def choose_boxsize(simulation):
    if (simulation.lower()=='massiveblackii'):
        return 100
    elif (simulation.lower()=='illustris'):
        return 75

def symmetrise_halo5(data, verbose=True, g=None, pivot='mass', central='spatial_central', snapshot=85, simulation='massiveblackii'):
    """
    Takes an array of objects associated with a particular halo,
    queries the database on coma to find the halo centroid positions
    and then applies a random rotation to each.
    The output is an array in the same format as given, but with
    new positions and shapes."""

    N = data['npart_baryon']
    n = data.size

    boxsize = choose_boxsize(simulation)

    # Work out the centroid about which to rotate  
    data, positions, (x0, y0, z0) = get_wrapped_positions(g, data, pivot=pivot, snapshot=snapshot, simulation=simulation, boxsize=boxsize) 
    cent = np.array([x0,y0,z0])

    # New array to store the output
    rot = np.zeros(data.size, dtype=data.dtype)  
    for name in data.dtype.names:
        rot[name][:n] = data[name]

    rot = utils.add_col(rot, 'x0', np.array([x0]*data.size), dtype=float)
    rot = utils.add_col(rot, 'y0', np.array([y0]*data.size), dtype=float)
    rot = utils.add_col(rot, 'z0', np.array([z0]*data.size), dtype=float)

    # Start the inner loop, over individual galaxies. Sorry.
    for i in xrange(n):

        # Cartesian galaxy position, relative to the halo centre
        pos = positions.T[i]

        # Galaxy position in spherical polar coordinates
        R = np.sqrt(sum(pos*pos))
        phi = np.arccos(pos[2]/R) * pos[0]/abs(pos[0])
        theta = np.arcsin(pos[1]/R/np.sin(phi))

       # if not np.isclose(rot['rh'][i],R):
        #    import pdb ; pdb.set_trace()
        rot['rh'][i] = R

        # We can skip the rest for central objects
        if (data[central][i]==1):
            rot = check_wrapping(i, rot)
            #rot['x'][i]=x0
            #rot['y'][i]=y0
            #rot['z'][i]=z0
            if verbose:
                print('skipping object -- it is classified as a central galaxy')
            continue

        if verbose:
            print(i,)

        # Choose a random point on a sphere about the centroid of radius R
        rotated = sample_sphere(1, norm=R, seed=None)
        rotated = np.array([rotated[0][0], rotated[1][0], rotated[2][0]])
        rot['x'][i] = rotated[0]+x0
        rot['y'][i] = rotated[1]+y0
        rot['z'][i] = rotated[2]+z0

        # Then work backwards to get the rotation matrix needed to transform the initial position to the rotated one
        rotation_axis, rotation_angle = infer_rotation_angle(pos,rotated)
        Rxyz = build_rotation_matrix(alpha=rotation_angle, vec=rotation_axis)

        if verbose:
            print('New position (x, y, z) : %3.3f, %3.3f %3.3f'%(rot['x'][i],rot['y'][i],rot['z'][i]))

        # Apply the same rotation to the three orientation vectors
        a3d = np.array([data['a1'][i], data['a2'][i], data['a3'][i]])
        b3d = np.array([data['b1'][i], data['b2'][i], data['b3'][i]])
        c3d = np.array([data['c1'][i], data['c2'][i], data['c3'][i]])
        arot = np.dot(Rxyz,a3d)*-1
        brot = np.dot(Rxyz,b3d)*-1
        crot = np.dot(Rxyz,c3d)*-1
        axes = {'a':arot,'b':brot,'c':crot}

        # And for the dark matter component
        a3d_dm = np.array([data['a1_dm'][i], data['a2_dm'][i], data['a3_dm'][i]])
        b3d_dm = np.array([data['b1_dm'][i], data['b2_dm'][i], data['b3_dm'][i]])
        c3d_dm = np.array([data['c1_dm'][i], data['c2_dm'][i], data['c3_dm'][i]])
        arot_dm = np.dot(Rxyz,a3d_dm)*-1
        brot_dm = np.dot(Rxyz,b3d_dm)*-1
        crot_dm = np.dot(Rxyz,c3d_dm)*-1
        axes_dm = {'a':arot_dm,'b':brot_dm,'c':crot_dm}

        # Stays the same
        rot['npart_baryon'][i] = data['npart_baryon'][i]

        # We also need to shift a few objects near the edges, which the rotation leaves outside the simulation box,
        # back to the other side of the universe 
        if verbose: 
            print('Rotation leaves %d object(s) outside the simulation box.'%(rot['x'][(rot['x']<0) | (rot['y']<0) | (rot['z']<0)].size))

        rot = check_wrapping(i, rot)

        # Store everything in the appropriate place
        for j in xrange(3):
            for axis in ['a','b','c']:
                rot['%c%d'%(axis,j+1)][i] =  axes[axis][j]
                rot['%c%d_dm'%(axis,j+1)][i] =  axes_dm[axis][j]

        # Work out the projected 2D ellipticities for the stellar component
        rot = project_ellipticities(i, rot, suffix='')
        # Same for dark matter 
        rot = project_ellipticities(i, rot, suffix='dm')


        
    return rot

def project_ellipticities(i, rot, suffix=''):

	if (suffix!=''):
		suffix = '_%s'%suffix

	a3d = np.array([[rot['a1'+suffix][i], rot['a2'+suffix][i], rot['a3'+suffix][i]]])
	b3d = np.array([[rot['b1'+suffix][i], rot['b2'+suffix][i], rot['b3'+suffix][i]]])
	c3d = np.array([[rot['c1'+suffix][i], rot['c2'+suffix][i], rot['c3'+suffix][i]]])
	q3d = np.array([np.sqrt(rot['lambda_a'+suffix][i]/rot['lambda_c'+suffix][i])])
	s3d = np.array([np.sqrt(rot['lambda_b'+suffix][i]/rot['lambda_c'+suffix][i])])

	e1,e2 = utils.project_3d_shape(a3d, b3d, c3d, q3d, s3d)
	rot['e1'+suffix][i] = e1
	rot['e2'+suffix][i] = e2

	return rot


def get_wrapped_positions(g, data, pivot='mass', snapshot=85, simulation='massiveblackii', boxsize=100):
    # Query the DB for the halo centre
    if (pivot.lower()=='mass'):
    	info = find_centre(g, snapshot=snapshot, simulation=simulation)
    elif (pivot.lower()=='most_massive_galaxy'):
        info = np.zeros(1,dtype=[('x',float),('y',float),('z',float)])
        mask = (data['most_massive']==1)
        if len(data['x'][mask])==1:
            info['x'] = data['x'][mask][0] * 1000
            info['y'] = data['y'][mask][0] * 1000
            info['z'] = data['z'][mask][0] * 1000
        else:
            info = find_centre(g, snapshot=snapshot, simulation=simulation)
            print('No central galaxy. Falling back on the centre of mass from the database')
            info = find_centre(g)
    elif (pivot.lower()=='most_central_galaxy'):
        info = np.zeros(1,dtype=[('x',float),('y',float),('z',float)])
        mask = (data['halo_id']==g)
        info['x'] = data['xcent'][mask][0] * 1000
        info['y'] = data['ycent'][mask][0] * 1000
        info['z'] = data['zcent'][mask][0] * 1000
    elif (pivot.lower()=='hybrid_central_galaxy'):
        info = np.zeros(1,dtype=[('x',float),('y',float),('z',float)])
        mask = (data['halo_id']==g) & (data['hybrid_central']==1)
        if (len(data['halo_id'][mask])==0):
            print('No central galaxy. Falling back on the centre of mass from the database')
            info = find_centre(g, snapshot=snapshot, simulation=simulation)
        else:
            info['x'] = data['x'][mask][0] * 1000
            info['y'] = data['y'][mask][0] * 1000
            info['z'] = data['z'][mask][0] * 1000
    else:
        raise ValueError('Unknown pivot option:%s'%pivot)

    
    for comp in ['x','y','z']:
        if (data[comp]<5).max() and (data[comp]>boxsize-5).max():
            # Shift everything back down to the lower edge of the simulation box
            # This will produce negative positions, which is fine if we shift the
            # galaxies outside the box back after the rotation

            # The 5 Mpc border is a bit arbitary, but it's probably safe to say any
            # halo with galaxies within both the uppermost and lowermost 5 Mpc is
            # an artefact of the periodic boundary conditions   
            data[comp][data[comp]>(boxsize-5)]-=boxsize


        # This should handle the case where the centroid is on the opposite
        # side of the box to all of the galaxies surviving cuts 
        if ( ((info[comp]/1000)>boxsize-5) and (data[comp]<5).max() ):
            info[comp]-=boxsize*1000
        if ( ((info[comp]/1000)<5) and (data[comp]>(boxsize-5)).max() ):
            info[comp]+=boxsize*1000


    x0,y0,z0 = info['x']/1000., info['y']/1000., info['z']/1000.
    positions = np.array([data['x']-x0, data['y']-y0, data['z']-z0])

    x0 = np.atleast_1d(x0)[0]
    y0 = np.atleast_1d(y0)[0]
    z0 = np.atleast_1d(z0)[0]

    return data, positions, (x0,y0,z0)

def sample_sphere(npoints, norm=1, seed=None):
    """Generates random points on a sphere of radius R"""
    if (seed!=None):
        np.random.seed(seed)
    u1 = np.random.rand(npoints)
    u2 = np.random.rand(npoints)

    # Define a vector position
    theta = np.arccos(2*u1-1)
    phi = 2 * np.pi * u2

    x = np.sin(theta)*np.cos(phi)
    y = np.sin(theta)*np.sin(phi)
    z = np.cos(theta)
    vec= np.array([x,y,z])
    vec *= norm

    return vec

def infer_rotation_angle(position0, position):
    """Work out two rotation angles that will transform one given 
    3D vector into another."""

    # Work out the unit vector orthogonal to the plane defined by the two position vectors
    rotation_axis = np.cross(position0, position)
    rotation_axis /= utils.get_norm(rotation_axis)

    # Then calculate the angle between the two position vectors in the 2D plane defined by them
    alpha = np.arccos( np.dot(position/utils.get_norm(position), position0/utils.get_norm(position0)) )
    cross = np.cross(position, position0);

    #alpha = -alpha

    return rotation_axis, alpha



def build_rotation_matrix(theta=None, phi=None, alpha=None, vec=[]):
    """Generates a random rotation matrix, 
    which transforms a given 3D position vector to a random position on the sphere.
    Usage : rotated = R.unrotated"""
    u1 = np.random.rand()
    u2 = np.random.rand()

    # Use two random values from a uniform distribution to generate an axis about which to rotate
    if phi==None:
        phi = np.arccos(2*u1-1)
    if theta==None:
        theta = 2 * np.pi * u2

    # Work out a Cartesian unit vector defined by the rotation axis 
    if (len(vec)==0):
        x = np.sin(theta)*np.cos(phi)
        y = np.sin(theta)*np.sin(phi)
        z = np.cos(theta)
        vec= np.array([x,y,z])
    vec/=utils.get_norm(vec)

    # Now generate a random rotation angle about that axis
    if alpha==None:
        alpha = np.random.rand() * 2 * np.pi
    cosa = np.cos(alpha)
    sina = np.sin(alpha)
    ux = vec[0]
    uy = vec[1]
    uz = vec[2]

    # Construct the 3x3 rotation matrix
    R = np.array([ [ (cosa + ux*ux*(1-cosa)), (ux*uy*(1-cosa) - uz*sina), (ux*uz*(1-cosa) + uy*sina)],
    [uy*ux*(1-cosa) + uz*sina, cosa + uy*uy*(1-cosa), uy*uz*(1-cosa) - ux*sina],
    [uz*ux*(1-cosa) - uy*sina, uz*uy*(1-cosa) + ux*sina, cosa + uz*uz*(1-cosa) ]] )

    return R

cosmoparam = {'omega_m':0.275, 'sigma8':0.816, 'ns': 0.968, 'omega_b': 0.046, 'omega_de': 0.725, 'h': 0.701,'w': -1.0}
def find_r200(M):
    """Given an array of galaxy masses in units of h^-1 M_*
    compute the virial radius in h^-1 Mpc."""

    # convert to SI units
    Mt = sum(M)/cosmoparam['h']*1.989e30
    C = 7.7836e7

    # Value in metres
    R200 = C * Mt**(1/3.)

    # Convert to h^-1 Mpc
    R200/=3.086e22
    R200*=cosmoparam['h']

    return R200


def find_centre(g, snapshot=85, simulation='massiveblackii'):
    """Locate the mass centroid of a particular group of galaxies"""
    if (simulation=='massiveblackii'):
        import pymysql as mdb
        sql = 'SELECT x,y,z FROM subfind_groups WHERE groupId=%d AND snapnum=%d;'%(g, snapshot)

        sqlserver='localhost' ; user='flanusse' ; password='mysqlpass' ; dbname='mb2_hydro'
        unix_socket='/home/rmandelb.proj/flanusse/mysql/mysql.sock'
        db = mdb.connect(sqlserver, user, password, dbname, unix_socket=unix_socket)

        c = db.cursor()
        c.execute(sql)
        results = fromarrays(np.array(c.fetchall()).squeeze().T,names='x,y,z')
        return results
    elif (simulation=='illustris'):
        import illustris_python as il
        root = '/nfs/nas-0-1/vat/Illustris-1'
        info = il.groupcat.loadSingle(root,snapshot,haloID=g)['GroupPos']
        out = np.array(1, dtype=[('x', float), ('y', float), ('z', float)])
        out['x'] = info[0]
        out['y'] = info[1]
        out['z'] = info[2]
        return out

def check_wrapping(i, rot, boxsize=100):
	# Lower edge on each axis
	if (rot['x'][i]<0):
		rot['x'][i]+=boxsize
	if (rot['y'][i]<0):
		rot['y'][i]+=boxsize
	if (rot['z'][i]<0):
		rot['z'][i]+=boxsize

	# Upper edge on each axis
	if (rot['x'][i]>boxsize):
		rot['x'][i]-=boxsize
	if (rot['y'][i]>boxsize):
		rot['y'][i]-=boxsize
	if (rot['z'][i]>boxsize):
		rot['z'][i]-=boxsize

	return rot


def randomise_shapes(cat):

    print('Randomising shapes')
    for j, row in enumerate(cat):
        rotated = sample_sphere(1, norm=1, seed=None)
        rotated = np.array([rotated[0][0], rotated[1][0], rotated[2][0]])
        pos = np.array([row['a1'], row['a2'], row['a3']])

        rotation_axis, rotation_angle = infer_rotation_angle(pos,rotated)
        Rxyz = build_rotation_matrix(alpha=rotation_angle, vec=rotation_axis)

        a3d = np.array([row['a1'], row['a2'], row['a3']])
        b3d = np.array([row['b1'], row['b2'], row['b3']])
        c3d = np.array([row['c1'], row['c2'], row['c3']])
        arot = np.dot(Rxyz,a3d)*-1
        brot = np.dot(Rxyz,b3d)*-1
        crot = np.dot(Rxyz,c3d)*-1
        cat['a1'][j] = arot[0]
        cat['a2'][j] = arot[1]
        cat['a3'][j] = arot[2]
        cat['b1'][j] = brot[0]
        cat['b2'][j] = brot[1]
        cat['b3'][j] = brot[2]
        cat['c1'][j] = crot[0]
        cat['c2'][j] = crot[1]
        cat['c3'][j] = crot[2]

        cat = project_ellipticities(j, cat, suffix='')

    return cat