Skip to content

Commit

Permalink
[TD-5007]<fix>: use build directory's taosdump to test.
Browse files Browse the repository at this point in the history 
btw, solve two taosdemo test cases too.
  • Loading branch information
@sangshuduo
sangshuduo committed Jul 1, 2021
1 parent 02051bf commit ecfb59f
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 56 deletions.
83 changes: 62 additions & 21 deletions tests/pytest/tools/taosdemoPerformance.py
View file
Expand Up @@ -17,10 +17,11 @@
import os.path
import json


class taosdemoPerformace:
def __init__(self, commitID, dbName):
self.commitID = commitID
self.dbName = dbName
self.dbName = dbName
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
Expand All @@ -30,8 +31,8 @@ def __init__(self, commitID, dbName):
self.user,
self.password,
self.config)
self.insertDB = "insertDB";
self.insertDB = "insertDB"

def generateJson(self):
db = {
"name": "%s" % self.insertDB,
Expand All @@ -41,7 +42,7 @@ def generateJson(self):

stb = {
"name": "meters",
"child_table_exists":"no",
"child_table_exists": "no",
"childtable_count": 10000,
"childtable_prefix": "stb_",
"auto_create_table": "no",
Expand All @@ -57,12 +58,12 @@ def generateJson(self):
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"tags_file": "",
"columns": [
{"type": "INT", "count": 4}
],
],
"tags": [
{"type": "INT", "count":1},
{"type": "INT", "count": 1},
{"type": "BINARY", "len": 16}
]
}
Expand All @@ -88,7 +89,7 @@ def generateJson(self):
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"num_of_records_per_req": 30000,
"databases": [db]
"databases": [db]
}

insert_json_file = f"/tmp/insert.json"
Expand All @@ -103,24 +104,56 @@ def getCMDOutput(self, cmd):
cmd.close()
return output

def insertData(self):
os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson())
self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'")
self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'")
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))

if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]

for root, dirs, files in os.walk(projPath):
if ("taosdemo" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath

def insertData(self):
tdSql.prepare()
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosdemo not found!")
else:
tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/"

os.system(
"%staosdemo -f %s > taosdemoperf.txt 2>&1" %
(binPath, self.generateJson()))
self.createTableTime = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
self.insertRecordsTime = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'")
self.recordsPerSecond = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'")
self.commitID = self.getCMDOutput("git rev-parse --short HEAD")
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'")
delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $4}'")
self.avgDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'")
delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $6}'")
self.maxDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'")
delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $8}'")
self.minDelay = delay[:-3]

os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt")

def createTablesAndStoreData(self):
cursor = self.conn.cursor()

cursor.execute("create database if not exists %s" % self.dbName)
cursor.execute("use %s" % self.dbName)
cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)")
Expand All @@ -130,13 +163,21 @@ def createTablesAndStoreData(self):
print("records per second: %f" % float(self.recordsPerSecond))
print("avg delay: %f" % float(self.avgDelay))
print("max delay: %f" % float(self.maxDelay))
print("min delay: %f" % float(self.minDelay))
cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" %
(float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay)))
print("min delay: %f" % float(self.minDelay))
cursor.execute(
"insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" %
(float(
self.createTableTime), float(
self.insertRecordsTime), float(
self.recordsPerSecond), self.commitID, float(
self.avgDelay), float(
self.maxDelay), float(
self.minDelay)))
cursor.execute("drop database if exists %s" % self.insertDB)

cursor.close()


if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
Expand All @@ -155,6 +196,6 @@ def createTablesAndStoreData(self):

args = parser.parse_args()

perftest = taosdemoPerformace(args.commit_id, args.database_name)
perftest = taosdemoPerformace(args.commit_id, args.database_name)
perftest.insertData()
perftest.createTablesAndStoreData()
6 changes: 3 additions & 3 deletions tests/pytest/tools/taosdemoTest.py
View file
Expand Up @@ -36,7 +36,7 @@ def getBuildPath(self):
projPath = selfPath[:selfPath.find("tests")]

for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
if ("taosdemo" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
Expand All @@ -47,9 +47,9 @@ def run(self):
tdSql.prepare()
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
tdLog.exit("taosdemo not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
os.system("%staosdemo -y -t %d -n %d" %
(binPath, self.numberOfTables, self.numberOfRecords))
Expand Down
94 changes: 62 additions & 32 deletions tests/pytest/tools/taosdumpTest.py
View file
Expand Up @@ -23,51 +23,79 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)

self.ts = 1538548685000
self.numberOfTables = 10000
self.numberOfRecords = 100

def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))

if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]

for root, dirs, files in os.walk(projPath):
if ("taosdump" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath

def run(self):
if not os.path.exists("/taosdumptest/tmp1"):
os.makedirs("/taosdumptest/tmp1")
if not os.path.exists("./taosdumptest/tmp1"):
os.makedirs("./taosdumptest/tmp1")
else:
print("目录存在")
if not os.path.exists("/taosdumptest/tmp2"):
os.makedirs("/taosdumptest/tmp2")

if not os.path.exists("./taosdumptest/tmp2"):
os.makedirs("./taosdumptest/tmp2")
tdSql.execute("drop database if exists db")
tdSql.execute("create database db days 11 keep 3649 blocks 8 ")
tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ")
tdSql.execute("use db")
tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("create table t1 using st tags(1, 'beijing')")
tdSql.execute(
"create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("create table t1 using st tags(1, 'beijing')")
sql = "insert into t1 values"
currts = self.ts
for i in range(100):
sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
tdSql.execute(sql)
tdSql.execute("create table t2 using st tags(2, 'shanghai')")
tdSql.execute("create table t2 using st tags(2, 'shanghai')")
sql = "insert into t2 values"
currts = self.ts
for i in range(100):
sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
tdSql.execute(sql)
os.system("taosdump --databases db -o /taosdumptest/tmp1")
os.system("taosdump --databases db1 -N -o /taosdumptest/tmp2")

tdSql.execute(sql)

buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosdemo not found!")
else:
tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/"

os.system("%staosdump --databases db -o ./taosdumptest/tmp1" % binPath)
os.system(
"%staosdump --databases db1 -N -o ./taosdumptest/tmp2" %
binPath)

tdSql.execute("drop database db")
tdSql.execute("drop database db1")
tdSql.query("show databases")
tdSql.checkRows(0)
os.system("taosdump -i /taosdumptest/tmp1")
os.system("taosdump -i /taosdumptest/tmp2")
os.system("%staosdump -i ./taosdumptest/tmp1" % binPath)
os.system("%staosdump -i ./taosdumptest/tmp2" % binPath)

tdSql.execute("use db")
tdSql.query("show databases")
tdSql.checkRows(2)
dbresult = tdSql.queryResult
# 6--days,7--keep0,keep1,keep, 12--block,
for i in range(len(dbresult)):
for i in range(len(dbresult)):
if dbresult[i][0] == 'db':
print(dbresult[i])
print(type(dbresult[i][6]))
Expand Down Expand Up @@ -102,40 +130,42 @@ def run(self):
tdSql.checkData(i, 1, i)
tdSql.checkData(i, 2, "nchar%d" % i)

# drop all databases，boundary value testing. length(databasename)<=32;length(tablesname)<=192
# drop all databases，boundary value testing.
# length(databasename)<=32;length(tablesname)<=192
tdSql.execute("drop database db")
tdSql.execute("drop database db1")
os.system("rm -rf /taosdumptest/tmp1")
os.system("rm -rf /taosdumptest/tmp2")
os.makedirs("/taosdumptest/tmp1")
tdSql.execute("create database db12312313231231321312312312_323")
tdSql.error("create database db12312313231231321312312312_3231")
os.system("rm -rf ./taosdumptest/tmp1")
os.system("rm -rf ./taosdumptest/tmp2")
os.makedirs("./taosdumptest/tmp1")
tdSql.execute("create database db12312313231231321312312312_323")
tdSql.error("create database db12312313231231321312312312_3231")
tdSql.execute("use db12312313231231321312312312_323")
tdSql.execute("create stable st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.error("create stable st_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("create stable st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute(
"create stable st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.error("create stable st1(ts timestamp, c1 int, col2_012345678901234567890123456789012345678901234567890123456789 nchar(10)) tags(t1 int, t2 binary(10))")

tdSql.execute("select * from db12312313231231321312312312_323.st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9")
tdSql.error("create table t0_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9 using st tags(1, 'beijing')")
tdSql.execute("select * from db12312313231231321312312312_323.st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9")
tdSql.error("create table t0_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9 using st tags(1, 'beijing')")
tdSql.query("show stables")
tdSql.checkRows(2)
os.system("taosdump --databases db12312313231231321312312312_323 -o /taosdumptest/tmp1")
tdSql.execute("drop database db12312313231231321312312312_323")
os.system("taosdump -i /taosdumptest/tmp1")
os.system(
"%staosdump --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % binPath)
tdSql.execute("drop database db12312313231231321312312312_323")
os.system("%staosdump -i ./taosdumptest/tmp1" % binPath)
tdSql.execute("use db12312313231231321312312312_323")
tdSql.query("show stables")
tdSql.checkRows(2)
os.system("rm -rf /taosdumptest/tmp1")
os.system("rm -rf /taosdumptest/tmp2")
os.system("rm -rf ./taosdumptest/tmp1")
os.system("rm -rf ./taosdumptest/tmp2")
os.system("rm -rf ./dump_result.txt")
os.system("rm -rf ./db.csv")

os.system("rm -rf ./db.csv")

def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)


tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

0 comments on commit ecfb59f

Please sign in to comment.