Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Updates for HomeAssistant instead of SQL
  • Loading branch information
qcasey committed Aug 10, 2020
1 parent a81eecd commit e333b4b
Showing 1 changed file with 82 additions and 271 deletions.
353 changes: 82 additions & 271 deletions capture/pycap.py
@@ -1,298 +1,109 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import argparse
from os import path
from configparser import ConfigParser
import pcapy, re, os
import mysql.connector
from mysql.connector import errorcode
import datetime, time
from time import gmtime, strftime
import pcapy, re, sys, requests
from impacket.ImpactDecoder import *

# constants
PROGRAM_NAME = "pycap"
CONFIG_FILE_NAME = PROGRAM_NAME + ".conf"
MENU_MAIN_DESC = "Capture TCP traffic from a SunPower PV monitor, populating SP_RAW_PRODUCTION table in the mysql DB.\nCreated by Eric Hampshire (ehampshire@gmail.com)."
PROGRAM_VERSION = '1.0'
regex = re.compile(r'^([0-9]+)\t')

def main():
global MAC, logFileDir, dbUser, dbPassword, dbName, dbHost

# build main menu parser
parser = argparse.ArgumentParser(prog=PROGRAM_NAME, description=MENU_MAIN_DESC)
parser.add_argument('-V', '-v', '--version', action='version',
version='%(prog)s (version {})'.format(PROGRAM_VERSION))
parser.add_argument("-c", "-config", dest="config", default=None, required=False, help="Optional config path.")

# parse the user's args
args = parser.parse_args()
#
# Configuration
#

# get config path and load config object
config_path = get_config_path(args)
config = load_config(config_path)
HOME_ASSISTANT_LONG_LIVED_API_KEY = "ABCDEFG"
HOME_ASSISTANT_URL = "http://192.168.1.20:8123"
SUNPOWER_PORTAL_IP = "34.208.188.187" # I doubt this will change often, but my IP was different from @ehampshire's. Best to check this with wireshark
DEVICE = "eth0"

dbHost = config["defaults"]["dbHost"]
dbName = config["defaults"]["dbName"]
dbUser = config["defaults"]["dbUser"]
dbPassword = config["defaults"]["dbPassword"]
MAC = config["defaults"]["MAC"]
ipAddress = config["defaults"]["ipAddress"]
logFileDir = config["defaults"]["logFileDir"]
#
#
#

# Grab a list of interfaces that pcap is able to listen on.
# The current user will be able to listen from all returned interfaces,
# using open_live to open them.
max_bytes = 10000
promiscuous = True
read_timeout = 100 # in milliseconds
dev = "eth0"
ifs = pcapy.findalldevs()

# No interfaces available, abort.
if 0 == len(ifs):
print "You don't have enough permissions to open any interface on this system."
sys.exit(1)

# Only one interface available, use it.
elif 1 == len(ifs):
print 'Only one interface present, defaulting to it.'
dev=ifs[0]

pc = pcapy.open_live(dev, max_bytes, promiscuous, read_timeout)

pc.setfilter('tcp')
pc.setfilter("ip && tcp && dst net %s && dst port 80" % (ipAddress))
#pc.setfilter("ip && tcp && dst net %s && dst port 80 && ether src %s" % (ipAddress, MAC))

packet_limit = -1 # infinite
pc.loop(packet_limit, recv_pkts) # capture packets
# Constants
regex = re.compile(r'^([0-9]+)\t')
head = {'Authorization': 'Bearer ' + HOME_ASSISTANT_LONG_LIVED_API_KEY}
maxBytes = 100000
promiscuous = True
readTimeout = 1000 # in milliseconds

def load_config(config_path):
# configure the config parser
config = ConfigParser()
if config_path is not None:
config.read(config_path)
return config
# Globals
lastProduction = None
lastActualProduction = None
lastConsumption = None

def get_config_path(args):
# if the config file is provided via command line arg then always use that
if args.config is not None:
return args.config
def is_number(s):
try:
float(s)
return True
except ValueError:
return False

# otherwise look for a config file path
config_file = None
config_home = path.join(path.expanduser("~/.config/pycap.conf"), CONFIG_FILE_NAME)
config_etc = path.join("/etc", CONFIG_FILE_NAME)
def http_post(url, data):
post = urlencode(data)
req = urllib2.Request(url, post)
response = urllib2.urlopen(req)
return response.read()

# conf file search order:
# 1. search the executable directory
# 2. search the user's home directory
# 3. search /etc
if path.isfile(CONFIG_FILE_NAME):
config_file = path.join(".", CONFIG_FILE_NAME)
elif path.isfile(config_home):
config_file = config_home
elif path.isfile(config_etc):
config_file = config_etc
return config_file
def parseAndPostData(msg):
global lastProduction, lastConsumption

def getLoggingTime():
return datetime.datetime.now().strftime("%Y%m%d")
if (msg[0] == "130"):
currentProduction = float(msg[4])

def appendLog(line):
global logFileDir
if not line:
return
this_line = line.split()
if (len(this_line) < 1):
return
if (this_line[0] == "POST" or this_line[0] == "Host:" or this_line[0] == "Content-Type:" or this_line[0] == "Content-Length:"):
return
logFileName = logFileDir + getLoggingTime() + "-py.txt"
#logFileName = logFileDir + getLoggingTime() + ".txt"
#print "logFileName: %s" % (logFileName)
with open(logFileName, 'a') as the_file:
last_line = tail(logFileName)
insert_line = last_line.split()
candidate = False
if (len(insert_line) < 14):
if (len(insert_line) < 1):
candidate = False
elif (insert_line[0] == "140"):
if (len(insert_line) < 3):
candidate = False
elif (len(insert_line) > 12 or insert_line[2] == "PVS5M508095p" or insert_line[2] == "PVS5M508095c"):
candidate = False
else:
if (this_line[0] != "130" or this_line[0] != "140"):
candidate = True
elif (insert_line[0] == "130"):
if (this_line[0] != "130" and this_line[0] != "140"):
candidate = True
if (candidate):
last_line = last_line.replace('\r\n', '').rstrip()
print 'last_line: %s\nthis_line: %s' % (last_line, line)
new_line = last_line + line.lstrip()
the_file.write('%s\n' % (new_line))
#the_file.write('%s\n' % (line))
insertLineIntoDb(new_line)
else:
the_file.write('%s\n' % (line))
insertLineIntoDb(line)
the_file.close()
if lastProduction:
actualProduction = currentProduction - lastProduction
print('\tActual Production (kWh): %.2f\n' % actualProduction)
response = requests.post(HOME_ASSISTANT_URL + "/api/states/sensor.power_production", json={"state": round(actualProduction, 3), "attributes": { "friendly_name": "Power Production", "unit_of_measurement": "kW", "icon": "hass:solar-power" }}, headers=head)
print('\tResponse: %s\n' % response.json())
lastActualProduction = actualProduction

def tail(filepath):
try:
filepath.is_file
fp = str(filepath)
except AttributeError:
fp = filepath
lastProduction = currentProduction

with open(fp, "rb") as f:
size = os.stat(fp).st_size
start_pos = 0 if size - 1 < 0 else size - 1
# Post temperature
response = requests.post(HOME_ASSISTANT_URL + "/api/states/sensor.inverter_temperature", json={"state": float(msg[10]), "attributes": { "friendly_name": "Inverter Temperature", "unit_of_measurement": "°C", "icon": "hass:thermometer" }}, headers=head)
print('\tTemperature Response: %s\n' % response.json())

if start_pos != 0:
f.seek(start_pos)
char = f.read(1)
elif (msg[0] == "131"):
currentConsumption = float(msg[5])

if char == b"\n":
start_pos -= 1
f.seek(start_pos)
if lastActualProduction and lastConsumption:
actualConsumption = currentConsumption - lastConsumption + lastActualProduction
print('\tActual Consumption (kWh): %.2f\n' % actualConsumption)
response = requests.post(HOME_ASSISTANT_URL + "/api/states/sensor.power_consumption", json={"state": round(actualConsumption, 3), "attributes": { "friendly_name": "Power Consumption", "unit_of_measurement": "kW", "icon": "hass:power-plug" }}, headers=head)
print('\tResponse: %s\n' % response.json())

if start_pos == 0:
f.seek(start_pos)
else:
char = ""
lastConsumption = currentConsumption

for pos in range(start_pos, -1, -1):
f.seek(pos)
# callback for received packets
def recv_pkts(hdr, data):
global regex

char = f.read(1)
line_list = EthDecoder().decode(data).child().child().get_data_as_string().split('\n')
for line in line_list:
if not line:
continue

if char == b"\n":
break
if regex.match(line) or len(line.strip()) > 0:
msg = line.split()

return f.readline()
# Ignore empty, header, or control messages
if (len(msg) < 1 or not is_number(msg[0]) or msg[0] == "100" or msg[0] == "102" or msg[0] == "1002" or msg[0] == "120"):
continue

# callback for received packets
def recv_pkts(hdr, data):
global regex
packet = EthDecoder().decode(data)
#print packet
ip = packet.child()
tcp = ip.child()
#print tcp
adata = tcp.get_data_as_string()
#adata = adata.replace('\r\n', '\r\n###~~~###')
arrline = adata.split('\n')
#print arrline
counter = 0;
for line in arrline:
#print "%s: %s" % (counter, line)
counter = counter + 1
matcher = regex.match(line)
if matcher:
msg = matcher.group().strip()
#print "msg: %s" % (msg)
if (msg == "140"): # this is a net metering message, and $value is net metering value in (IIRC) W averaged over the 5-minute interval
# consumption = (corresponding production) + net
print "net metering message (140):\n\t%s\n" % (line)
appendLog(line)
elif (msg == "130"): # this is a production message, and $value is a production valuein (IIRC) W averaged over the 5-minute interval
print "production message (130):\n\t%s\n" % (line)
appendLog(line)
elif (msg == "100"): # this is a control message / keep-alive
print "control message (100):\n\t%s\n" % (line)
elif (msg == "102"): # this is a checksum message
print "checksum message (102):\n\t%s\n" % (line)
elif (msg == "141"): # this is a consumption message?
print "message (141):\n\t%s\n" % (line)
elif (msg == "120"): # this is a consumption message?
print "message (120):\n\t%s\n" % (line)
else:
print "unknown message:\n\t%s\n" % (line)
appendLog(line)
else:
print "unmatched message:\n\t%s\n" % (line)
appendLog(line)
if(msg[0] == "130"):
print("production message (130): ")
elif(msg[0] == "131"):
print("net metering message (131): ")
else:
print("unmatched message: ")

def insertLineIntoDb(line):
global dbUser, dbPassword, dbHost, dbName
if (dbHost is None or dbName is None or dbUser is None or dbPassword is None):
print("Something is wrong with your DB config, please specify the config file or check your settings.")
sys.exit(1)
try:
cnx = mysql.connector.connect(user=dbUser, password=dbPassword, host=dbHost, database=dbName)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
print("Something is wrong with your user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
print("Database does not exist")
else:
print(err)
else:
#print "We are logged in!"
add_raw_production_sql_130 = ("INSERT INTO sp_raw_production "
"(message_type, src_timestamp, device_serial, device_description, watts, v1, v2, v3, v4, v5, v6, v7, v8, v9) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)")
add_raw_production_sql_140 = ("INSERT INTO sp_raw_production "
"(message_type, src_timestamp, device_serial, device_description, v1, watts, v2, v3, v4, v5, v6, v7) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)")
add_raw_production_sql = add_raw_production_sql_130
cursor = cnx.cursor()
fcursor = cnx.cursor()
#example_data = "130 20170108161000 414051636007015 AC_Module_Type_C 18.7783 0.0144 245.3928 0.1376 0.017 55.3412 0.3274 5.5 60.0006 0"
#example_data = "140 20170108171000 PVS5M508095c PVS5M0400c 125 648.84 -0.1225 1.1217 1.4173 -0.0836 60.029 0"
#insert_line = example_data.split()
insert_line = line.split()
if (len(insert_line) < 14):
if (insert_line[0] == "140"):
if (len(insert_line) < 12):
return;
elif (insert_line[3] == "PVS5M0400p"):
return;
else:
add_raw_production_sql = add_raw_production_sql_140
else:
print "line missing values, skipping... insert_line: ", insert_line
return
#print insert_line
#print "attempting to parse date..."
new_timestamp = datetime.datetime.strptime(insert_line[1], "%Y%m%d%H%M%S")
#print new_timestamp
insert_line[1] = new_timestamp
check_existing_record_query = ("SELECT id FROM `sp_raw_production` WHERE src_timestamp=%s and device_serial=%s")
check_existing_record_data = (insert_line[1], insert_line[2])
try:
fcursor.execute(check_existing_record_query, check_existing_record_data)
#print(fcursor.statement)
except:
print(fcursor.statement)
raise

if (fcursor.with_rows):
id = fcursor.fetchone()
if (id is None):
try:
cursor.execute(add_raw_production_sql, insert_line)
print(cursor.statement)
except:
print(insert_line)
print(cursor.statement)
raise
id = cursor.lastrowid
print "Insert SUCCESS! TableID: ", id
else:
print "Row exists ", id," not adding!"
print('\t%s' % msg)

#print "Logging out of the DB!"
cnx.commit()
cursor.close()
cnx.close()
if(msg[0] == "130" or msg[0] == "131"):
try:
parseAndPostData(msg)
except Exception as e:
print("Failed to parse message, stumbled on exception: \n%s" % e)

if __name__ == "__main__":
sys.exit(main())
pc = pcapy.open_live(DEVICE, maxBytes, promiscuous, readTimeout)
pc.setfilter("ip && tcp")
sys.exit(pc.loop(-1, recv_pkts)) # capture packets

0 comments on commit e333b4b

Please sign in to comment.