Activation.obn

MODULE Activation;

	IMPORT Math, RandomNumbers;
	
	TYPE
    	DropoutVec* = ARRAY 100 OF REAL;	

	PROCEDURE ReLU*(input: REAL): REAL;

	VAR z: REAL;
		
	BEGIN
	
		IF (input > 0.0) THEN
			z := input
		ELSE
			z := 0.0
		END;
		
		RETURN z (* value of the function *)
	
	END ReLU;
	
	
	
	PROCEDURE Sigmoid*(input: REAL): REAL;

	VAR z: REAL;
		
	BEGIN
	
		z := 1.0/(1.0+Math.exp(-input));
		
		RETURN z (* value of the function *)
	
	END Sigmoid;
	
	
	
	PROCEDURE Dropout*(VAR A: DropoutVec; N: INTEGER; ratio: REAL);

	VAR
		i, j : LONGINT;
		tmp : REAL;
		
	BEGIN
	
		FOR i := 1 TO N DO A[i] := 0.0 END;
	
		FOR i := 1 TO ENTIER(N*(1.0-ratio))-1 DO		
			A[i] := 1.0/(1.0-ratio);
		END;
		
		FOR i := 1 TO N DO
			j := RandomNumbers.Int(N);
			IF (j >= 1) THEN
				IF (j <= N) THEN
					tmp := A[i]; A[i] := A[j]; A[j] := tmp
				END
			END
		END
	
	END Dropout;		
	
END Activation.