EfergyRPI_mysql.c

/*---------------------------------------------------------------------

EFERGY ENERGY SENSOR RTL-SDR DECODER via rtl_fm
(Formerly named EfergyRPI_001.c )

Copyright 2013 Nathaniel Elijah

Permission is hereby granted to use this Software for any purpose
including combining with commercial products, creating derivative
works, and redistribution of source or binary code, without
limitation or consideration. Any redistributed copies of this
Software must include the above Copyright Notice.

THIS SOFTWARE IS PROVIDED "AS IS". THE AUTHOR OF THIS CODE MAKES NO
WARRANTIES REGARDING THIS SOFTWARE, EXPRESS OR IMPLIED, AS TO ITS
SUITABILITY OR FITNESS FOR A PARTICULAR PURPOSE.

Compile:

gcc -lm -o EfergyDecode EfergyRPI_log.c

Execute using the following parameters:

rtl_fm -f 433550000 -s 200000 -r 96000 -g 19.7 2>/dev/null | ./EfergyRPI_log
--------------------------------------------------------------------- */
// Trivial Modifications for Data Logging by Gough (me@goughlui.com)
//
// Suggested compilation command is:
// gcc -O3 -o EfergyRPI_001 EfergyRPI_001.c -lm
// Note: -lm coming last avoids error from compiler which complains of
// undefined reference to function `pow'. The addition of -O3 turns on
// compiler optimizations which may or may not improve performance.
//
// Logging filename defined by additional command line argument.
//
// Added logging to file by append, with adjustable "samples to flush".
// This can avoid SSD wear while also avoiding too much lost data in case
// of power loss or crash.
//
// Also have ability to change line endings to DOS \r\n format.
//
// Consider changing the Voltage to match your local line voltage for 
// best results.
//
// Poor signals seem to cause rtl_fm and this code to consume CPU and slow
// down. Best results for me seem to be with the E4000 tuner. My R820T
// requires higher gain but sees some decode issues. Sometimes it stops
// altogether, but that's because rtl_fm seems to starve in providing
// samples.
//
// Should build and run on anything where rtl-fm works. You'll need to grab
// build, and install rtl-sdr from http://sdr.osmocom.org/trac/wiki/rtl-sdr
// first.
//
// Execute similarly
// rtl_fm -f 433550000 -s 200000 -r 96000 -g 19.7 2>/dev/null | ./EfergyRPI_log
// but play with the value for gain (in this case, 19.7) to achieve best result.
// -------------------------------------------------------------------------------------------------
// Changes by github user magellannh
//
// 08-14-2014  - Changed logic to support different Efergy sensor types without changing settings
//                         Logic uses message size to determine if checksum or crc should be used.
//                         Also, changed code so instead of doing bit by bit processing when samples arrive, 
//                         a frame's worth of samples is saved after a valid preamble and then processed.
//                         One advantage of this approach is that the center is recalculated for each frame
//                         for more reliable decoding, especially if there's lots of 433Mhz noise,
//                         which for me was throwing off the old center calculation (noise from weather sensors).
//                         This is less efficient, but still uses < 7% of cpu on raspberry pi
//                          (plus another 25% for rtl_fm w/ -A fast option)
//                         
// 08-13-2014  - Added code to check the CRC-CCIT (Xmodem)  crc used by Elite 3.0 TPM
// 08/12/2014 - Some debugging and sample analysis code added 
//
// Bug Fix -	Changed frame bytearray to unsigned char and added cast on  byte used in pow() function
//	to explicitly make it signed char.  Default signed/unsigned for char is compiler dependent
//	and this resulted in various problems.
// 
// New Feature  - Added frame analysis feature that dumps debug information to help characterize the FSK
//	sample data received by rtl_fm.  The feature is invoked using a "-d" option followed by
//	an optional debug level (1..4).  The output is sent to stdout. 
//
// Usage Examples: (original examples above still work as before plus these new options)
//
//	rtl_fm -f 433.51e6 -s 200000 -r 96000 -A fast  | ./EfergyRPI_log -d 1
//		This mode shows the least information which is just the best guess at the decoded frame and a KW calculation
//		using bytes 4, 5, and 6.  The checksum is computed and displayed, but not validated.
//	rtl_fm -f 433.51e6 -s 200000 -r 96000 -A fast  | ./EfergyRPI_log -d 2
//		This  level shows average plus and minus sample values and centering which can help with finding the best frequency. 
//		Adjust frequency to get wave center close to  0 .  If center is too high, lower frequency, otherwise increase it.
//	rtl_fm -f 433.51e6 -s 200000 -r 96000 -A fast  | ./EfergyRPI_log -d 3
//		This mode outputs a summary with counts of consecutive positive or negative samples.  These consecutive pulse counts
//		are what the main code uses to decode 0 and 1 data bits.  See comments below for more details on expected pulse counts.
//	rtl_fm -f 433.51e6 -s 200000 -r 96000 -A fast  | ./EfergyRPI_log -d 4
//		This mode shows everything in modes 1..3 plus a raw dump of the sample data received from rtl_fm.
//
//	*Notice the "-A fast" option on  rtl_fm.  This cut Raspberry Pi cpu load from 50% to 25% and decode still worked fine.
//	Also, with an R820T USB dongle, leaving  rtl_fm gain in 'auto' mode  produced the best results.
//
#include <mysql.h>
#include <stdio.h>
#include <stdint.h>
#include <time.h>
#include <math.h>
#include <stdlib.h> // For exit function
#include <string.h>
#include <sys/time.h>


#define VOLTAGE			240	/*For non-TPM type sensors, set to line voltage */
//#define VOLTAGE			1	/* For Efergy Elite 3.0 TPM,  set to 1 */

#define FRAMEBYTECOUNT				9  /* Attempt to decode up to this many bytes.   */
#define MINLOWBITS				3  /* Min number of positive samples for a logic 0 */
#define MINHIGHBITS				9  /* Min number of positive samples for a logic 1 */
#define MIN_POSITIVE_PREAMBLE_SAMPLES		40 /* Number of positive samples in  an efergy  preamble */
#define MIN_NEGATIVE_PREAMBLE_SAMPLES		40 /* Number of negative samples for a valid preamble  */
#define EXPECTED_BYTECOUNT_IF_CHECKSUM_USED	8
#define EXPECTED_BYTECOUNT_IF_CRC_USED 		9

float watts = 0;
time_t abans;
int analysis_wavecenter;

#define LOGTYPE			1	// Allows changing line-endings - 0 is for Unix /n, 1 for Windows /r/n
#define SAMPLES_TO_FLUSH	10	// Number of samples taken before writing to file.
					// Setting this too low will cause excessive wear to flash due to updates to
					// filesystem! You have been warned! Set to 10 samples for 6 seconds = every min.					
int loggingok;	 // Global var indicating logging on or off
int samplecount; // Global var counter for samples taken since last flush
FILE *fp;	 // Global var file handle

// Instead of processing frames bit by bit as samples arrive from rtl_fm, all samples are stored once a preamble is detected until
// enough samples have been saved to cover the expected maximum frame size.  This maximum number of samples needed for a frame
// is an estimate with padding which will hopefully be enough to store a full frame with some extra.
// 
// It seems that with most Eferg1ormats, each data bit is encoded  using some combination of about 18-20 rtl_fm samples.
// zero bits are usually received as 10-13 negative samples followed by 4-7 positive samples, while 1 bits
// come in as 4-7 negative samples followed by 10-13 positive samples ( these #s may have wide tolerences)
// If the signal has excessive noise, it's theoretically possible to fill up this storage and still have more frame data coming in.
// The code handles this overflow by trunkating the frame, but when this happens, it usually means the data is  junk anyway.
// 
// To skip over noise frames, the code checks for a sequence with both a positive and negative preamble back to back (can be pos-neg or neg-pos)
//  From empirical testing, the preamble is usually about 180 negative samples followed by 45-50 positive samples.
// 
// At some frequencies the sign of the sample data becomes perfectly inverted.  When this happens,  frame data can still be decoded by keying off
// negative pulse sequences instead of positive pulse sequences.  This inversion condition can be detected by looking at the sign of the first samples
// received after the preamble.  If the first set of samples is negative, the data can get decoded from the positive sample pulses.  If the first set of
//samples after the preamble is  greater than 0 then the data probably can be found by decoding negative samples using the same rules that are usually
// used when decoding with the positive samples.  The analysis code automatically checks for this 'inverted' signal condition and decodes from either
// the positive or negative pulse stream depending on that check.

#define APPROX_SAMPLES_PER_BIT  19
#define FRAMEBITCOUNT           (FRAMEBYTECOUNT*8)  /* bits for entire frame (not including preamble) */
#define SAMPLE_STORE_SIZE       (FRAMEBITCOUNT*APPROX_SAMPLES_PER_BIT)

int sample_storage[SAMPLE_STORE_SIZE];			
int sample_store_index;
   MYSQL *conn;
   MYSQL_RES *res;
   MYSQL_ROW row;
  /* Change me */
   char *server = "localhost";
   char *user = "root";
   char *password = "****";
   char *database = "openenergymonitor";


int decode_bytes_from_pulse_counts(int pulse_store[], int pulse_store_index, unsigned char bytes[]) {
	int i;
	int dbit=0;
	int bitpos=0;
	unsigned char bytedata=0;
	int bytecount=0;
	
	for (i=0;i<FRAMEBYTECOUNT;i++)
		bytes[i]=0;		
	for (i=0;i<pulse_store_index;i++) {
		if (pulse_store[i] > MINLOWBITS) {
			dbit++;
			bitpos++;	
			bytedata = bytedata << 1;
			if (pulse_store[i] > MINHIGHBITS)
				bytedata = bytedata | 0x1;
			if (bitpos > 7) {
				bytes[bytecount] = bytedata;
				bytedata = 0;
				bitpos = 0;
				bytecount++;
				if (bytecount == FRAMEBYTECOUNT) {
					return bytecount;
				}
			}
		}
	}
	return bytecount;
}

unsigned char compute_checksum(unsigned char bytes[], int bytecount) {
	// Calculate simple 1 byte checksum on message bytes
	unsigned char tbyte=0x00;
	int i;
	for(i=0;i<(bytecount-1);i++) {
	  tbyte += bytes[i];
	}
	return tbyte;
}

uint16_t compute_crc(unsigned char bytes[], int bytecount) {
	// Calculate  CRC-CCIT (Xmodem)  crc using 0x1021 polynomial
	uint16_t crc=0;
	int i;
	for (i=0;i<bytecount-2;i++) {
		crc = crc ^ ((uint16_t) bytes[i] << 8);
		int j;
		for (j=0; j<8; j++) {
			if (crc & 0x8000)
				crc = (crc << 1) ^ 0x1021;
			else
				crc <<= 1;
		}
	}
	return crc;
}

int calculate_wave_center(int *avg_positive_sample, int *avg_negative_sample) {
	int i;	
	int64_t avg_neg=0;
	int64_t avg_pos=0;
	int pos_count=0;
	int neg_count=0;
	for (i=0;i<sample_store_index;i++)
		if (sample_storage[i] >=0) {
			avg_pos += sample_storage[i];
			pos_count++;
		} else {
			avg_neg += sample_storage[i];
			neg_count++;
		}
	if (pos_count!=0) 
		avg_pos /= pos_count;
	if (neg_count!=0)
		avg_neg /= neg_count;
	*avg_positive_sample = avg_pos;
	*avg_negative_sample = avg_neg;
	int diff =(avg_neg + ((avg_pos-avg_neg)/2));
	return diff;
}

int generate_pulse_count_array(int display_pulse_details, int pulse_count_storage[]) {

	// From empirical analysis with an Elite 3.0 TPM transmitter, sometimes the data can be decoded by counting negative pulses rather than positive
	// pulses.  It seems that if the first sequence after the preamble is positive pulses, the data can be decoded by parsing using the negative pulse counts.
	// This flag decoder automatically detect this.
	int store_positive_pulses=(sample_storage[2] < analysis_wavecenter);

	if (display_pulse_details) printf("\nPulse stream for this frame (P-Consecutive samples > center, N-Consecutive samples < center)\n");

	int wrap_count=0;
	int pulse_count=0;
	int space_count=0;
	int pulse_store_index=0;
	int space_store_index=0;
	int display_pulse_info=1;
	int i;
	for(i=0;i<sample_store_index;i++) {
		int samplec = sample_storage[i] - analysis_wavecenter;
		if (samplec < 0) {
			if (pulse_count > 0) {
				if (store_positive_pulses)
				  pulse_count_storage[pulse_store_index++]=pulse_count;
				if (display_pulse_details) printf("%2dP ", pulse_count);
				wrap_count++;
			}
			pulse_count=0;
			space_count++;
		} else {
			if (space_count > 0) {
				if (store_positive_pulses==0)
				 pulse_count_storage[pulse_store_index++]=space_count;
				if (display_pulse_details) printf("%2dN ", space_count);
				wrap_count++;
			}
			space_count=0;
			pulse_count++;
		}
		if (wrap_count >= 16) {
			if (display_pulse_details) printf("\n");
			wrap_count=0;
		}
	}
	if (display_pulse_details) printf("\n\n");
	
	return pulse_store_index;
}



void display_frame_data(int debug_level, char *msg, unsigned char bytes[], int bytecount) {

 	time_t ltime; 
	char buffer[80];
	time( &ltime );
	struct tm *curtime = localtime( &ltime );
	//strftime(buffer,80,"%x,%X", curtime); 
strftime(buffer,80,"%F %X", curtime); 

	//printf("el dia el posa com a %u -> \n", (unsigned)time(&ltime));

	// Some magic here to figure out whether the message has a 1 byte checksum or 2 byte crc
	char *data_ok_str = (char *) 0;
	unsigned char checksum=0;
	uint16_t crc = 0;
	if (bytecount == EXPECTED_BYTECOUNT_IF_CHECKSUM_USED) {
		checksum = compute_checksum(bytes, bytecount);
		if (checksum == bytes[bytecount-1])
			data_ok_str = "chksum ok";
	} else if (bytecount == EXPECTED_BYTECOUNT_IF_CRC_USED) {
		crc = compute_crc(bytes, bytecount);
		if (crc == ((bytes[bytecount-2]<<8) | bytes[bytecount-1]))
			data_ok_str = "crc ok";
	}

	double current_adc = (bytes[4] * 256) + bytes[5];
	double result  = (VOLTAGE*current_adc) / ((double) (32768) / (double) pow(2,(signed char) bytes[6]));
	if (debug_level > 0) {
		if (debug_level == 1)
			printf("%s  %s ", buffer, msg);
		else
			printf("%s ", msg);
			
		int i;
		for(i=0;i<bytecount;i++)
			printf("%02x ",bytes[i]);
	
		if (data_ok_str != (char *) 0)
			printf(data_ok_str);
		else {
			checksum = compute_checksum(bytes, bytecount);
			crc = compute_crc(bytes, bytecount);
			printf(" cksum: %02x crc16: %04x ",checksum, crc);
		}
		if (result < 100)
			printf("  kW: %4.3f\n", result);
		else {
			printf("  kW: <out of range>\n");
			if (data_ok_str != (char *) 0)
			  printf("*For Efergy True Power Moniter (TPM), set VOLTAGE=1 before compiling\n");
		}		
        } else if (data_ok_str != (char *) 0) {
		
	int dif_w=(watts-result);
	if (dif_w<0) dif_w=dif_w*(-1);

if( ((watts+10) < result) || ((watts-10) > result)){
	//printf("diferencia watts %d - %f - %f \n",dif_w, watts,result);

	double diff_t=0.00;
	char *buf;
	size_t sz;
	struct tm *curtime2 = localtime( &abans );
	char buffer2[80];
	strftime(buffer2,80,"%F %X", curtime2); 

	diff_t = difftime(ltime,abans);
/*abans = abans - 10;
		struct tm *curtime3 = localtime( &abans );
	char buffer3[80];
	strftime(buffer3,80,"%F %X", curtime3); 
	printf("diferencia temps %f - %s - %s - %s \n",diff_t, buffer,buffer2,buffer3);*/
	//printf("diferencia temps %f - %s - %s \n",diff_t, buffer,buffer2);
	char *buf2;
	size_t sz2;
	
	if(diff_t>50 && (dif_w>20)){
		//printf("-diferencia temps %f - %s - %s \n",diff_t, buffer,buffer2);
		//printf("-diferencia watts %d - %f - %f \n",dif_w, watts,result);
		abans = ltime - 10;
		struct tm *curtime3 = localtime( &abans );
		char buffer3[80];
		strftime(buffer3,80,"%F %X", curtime3); 

		sz2 = snprintf(NULL, 0, "insert into consumreals(date, value) VALUES('%s','%f')\n",buffer3,watts);
		buf2 = (char *)malloc(sz2 + 1); 
		snprintf(buf2, sz2+1, "insert into consumreals(date, value) VALUES('%s','%f')\n",buffer3,watts);
	   if (mysql_query(conn, buf2)) {
	      fprintf(stderr, "%s\n", mysql_error(conn));
	   }

	}

	abans=ltime;
	//printf("diferencia temps %f = %s - %s\n",diff_t,buffer,buffer2);
	watts = result;

	/*
	sz = snprintf(NULL, 0, "insert into consumreals(date,timestamp, value) VALUES('%s','%u','%f')\n",buffer,(unsigned)time(&ltime),result);
	buf = (char *)malloc(sz + 1); 
	snprintf(buf, sz+1, "insert into consumreals(date,timestamp, value) VALUES('%s','%u','%f')\n",buffer,(unsigned)time(&ltime),result);
	*/
	sz = snprintf(NULL, 0, "insert into consumreals(date, value) VALUES('%s','%f')\n",buffer,result);
	buf = (char *)malloc(sz + 1); 
	snprintf(buf, sz+1, "insert into consumreals(date, value) VALUES('%s','%f')\n",buffer,result);
	//printf(buf);
	/* send SQL query */
	   if (mysql_query(conn, buf)) {
	      fprintf(stderr, "%s\n", mysql_error(conn));
	     // exit(1);
	   }
		//	printf("insert into consumreals(date, value) VALUES('%s','%f')\n",buffer,result);
		  /* res = mysql_use_result(conn);
	   printf("MySQL Tables in mysql database:\n");
	   while ((row = mysql_fetch_row(res)) != NULL)
	      printf("%s \n", row[0]);
	   mysql_free_result(res);*/
}

		if(loggingok) {
		  if(LOGTYPE) {
		    fprintf(fp,"%s,%f\r\n",buffer,result);
		  } else {
		    fprintf(fp,"%s,%f\n",buffer,result);
		  }
		  samplecount++;
		  if(samplecount==SAMPLES_TO_FLUSH) {
		    samplecount=0;
		    fflush(fp);
		  }
		}
		fflush(stdout);
	} /*else
		printf("Checksum/CEC Error.  Enable debug output with -d option\n");*/
}

void analyze_efergy_message(int debug_level) {

	// See how balanced/centered the sample data is.  Best case is  diff close to 0
	int avg_pos, avg_neg;
	int difference = calculate_wave_center(&avg_pos, &avg_neg);
		
	if (debug_level > 1) {
		time_t ltime; 
		char buffer[80];
		time( &ltime );
		struct tm *curtime = localtime( &ltime );
		strftime(buffer,80,"%x,%X", curtime); 		
		printf("\nAnalysis of rtl_fm sample data for frame received on %s\n", buffer);
		printf("     Number of Samples: %6d\n", sample_store_index);
		printf("    Avg. Sample Values: %6d (negative)   %6d (positive)\n", avg_neg, avg_pos);
		printf("           Wave Center: %6d (this frame) %6d (last frame)\n", difference, analysis_wavecenter);
	} 
	analysis_wavecenter = difference; // Use the calculated wave center from this sample to process next frame
	
	if (debug_level==4) { // Raw Sample Dump only in highest debug level
		int wrap_count=0;
		printf("\nShowing raw rtl_fm sample data received between start of frame and end of frame\n");
		int i;
		for(i=0;i<sample_store_index;i++) {
			printf("%6d ", sample_storage[i] - analysis_wavecenter);
			wrap_count++;
			if (wrap_count >= 16) {
				printf("\n");
				wrap_count=0;
			}
		}
		printf("\n\n");
	}

	int display_pulse_details = (debug_level >= 3?1:0);
	int pulse_count_storage[SAMPLE_STORE_SIZE];
	int pulse_store_index = generate_pulse_count_array(display_pulse_details, pulse_count_storage);
	unsigned char bytearray[FRAMEBYTECOUNT];
	int bytecount=decode_bytes_from_pulse_counts(pulse_count_storage, pulse_store_index, bytearray);
	char *frame_msg;
	if (sample_storage[2] < analysis_wavecenter)
		frame_msg = "Msg:";
	else
		frame_msg = "Msg (from negative pulses):";
	display_frame_data(debug_level, frame_msg, bytearray, bytecount);
	
	if (debug_level>1) printf("\n");
}

void  main (int argc, char**argv) 
{

time( &abans );
   conn = mysql_init(NULL);
   
   /* Connect to database */
   if (!mysql_real_connect(conn, server,
         user, password, database, 0, NULL, 0)) {
      fprintf(stderr, "%s\n", mysql_error(conn));
      exit(1);
   }

	int debug_level = 0;
	
	// Give rtl_fm program some time to get initialized so its startup messages don't interleave with ours
	sleep(1);	
	
	if ((argc==2) && (strncmp(argv[1], "-h", 2)==0)) {
	  printf("\nUsage: %s              - Normal mode\n",argv[0]);
	  printf("       %s <filename>   - Normal mode plus log samples to output file\n", argv[0]);
	  printf("       %s -d [1,2,3,4] - Set debug/verbosity.  Default level 0 has minimum output\n",argv[0]);
	  exit(0);
	} else if ((argc==3) && (strncmp(argv[1], "-d", 2)==0)) {
	  debug_level = strtol(argv[2], NULL, 0);
	  if ((debug_level < 1) || (debug_level > 4)) {
	      fprintf(stderr, "\nDebug level (-d option) must be between 1 and 4\n");
	      exit(EXIT_FAILURE);
	  }
	} else if ((argc==2) && (strcmp(argv[1], "-d")==0))
	  debug_level=3;
	else if (argc==2) {
	  fp = fopen(argv[1], "a"); // Log file opened in append mode to avoid destroying data
	  samplecount=0; // Reset sample counter
	  loggingok=1;
	  if (fp == NULL) {
	      fprintf(stderr, "\nFailed to open log file!\n");
	      exit(EXIT_FAILURE);
	  }
	} else {
	  loggingok=0;
	}

	if (debug_level > 0)
		printf("\nEfergy Power Monitor Decoder - (debug level %d)\n\n", debug_level);
	else
		printf("\nEfergy Energy Monitor Decoder\n\n");
		
	int prvsamp;
	analysis_wavecenter = 0;
	
	while( !feof(stdin) ) {

		// Look for a valid Efergy Preamble sequence which we'll define as
		// a sequence of at least MIN_PEAMBLE_SIZE positive and negative or negative and positive pulses. eg 50N+50P or 50P+50N
		int negative_preamble_count=0;
		int positive_preamble_count=0;
		prvsamp = 0;
		while ( !feof(stdin) ) {	
			int cursamp  = (int16_t) (fgetc(stdin) | fgetc(stdin)<<8);			
			// Check for preamble 
			if ((prvsamp >= analysis_wavecenter) && (cursamp >= analysis_wavecenter)) {
				positive_preamble_count++;
			} else if ((prvsamp < analysis_wavecenter) && (cursamp < analysis_wavecenter)) {
				negative_preamble_count++;				
			} else if ((prvsamp >= analysis_wavecenter) && (cursamp < analysis_wavecenter)) {
				if ((positive_preamble_count > MIN_POSITIVE_PREAMBLE_SAMPLES) &&
					(negative_preamble_count > MIN_NEGATIVE_PREAMBLE_SAMPLES))
					break;
				negative_preamble_count=0;
			} else if ((prvsamp < analysis_wavecenter) && (cursamp >= analysis_wavecenter)) {
				if ((positive_preamble_count > MIN_POSITIVE_PREAMBLE_SAMPLES) &&
					(negative_preamble_count > MIN_NEGATIVE_PREAMBLE_SAMPLES))
					break;
				positive_preamble_count=0;
			}	
			prvsamp = cursamp;
		} // end of find preamble while loop
			
		sample_store_index=0;
		while( !feof(stdin) ) {
			int cursamp  = (int16_t) (fgetc(stdin) | fgetc(stdin)<<8);
			sample_storage[sample_store_index] = cursamp;
			if (sample_store_index < (SAMPLE_STORE_SIZE-1))
				sample_store_index++;
			else {
				analyze_efergy_message(debug_level);
				break;
			}
		} // Frame processing while 
	} // outermost while 
	
	if(loggingok) {
		   mysql_close(conn);
	    fclose(fp); // If rtl-fm gives EOF and program terminates, close file gracefully.
	}
}