Skip to content
Permalink
Browse files

add query scripts (#850)

  • Loading branch information...
btellstrom authored and monperrus committed Jun 14, 2019
1 parent 54c8b09 commit 4fc0670315d739c1f9be037aaadd8b72df25ba77
Showing with 60 additions and 31 deletions.
  1. +21 −0 doc/options-configuration-tips.md
  2. +39 −31 repairnator/scripts/patched_builds.py
@@ -158,6 +158,27 @@ with the travis-build URL, the github commit URL along with every
patch that was created for this specific build together with the
toolname that produced each patch.

The text file will have a html format and the following xpath queries
provide some interesting data for each file.

### Select all patches created by "tool"
`//[@title="tool"]`

### Select all patches
`//[@class="patch"]`

### Select travis-url
`//[@id="travis-url"]`

### Select commit-url
`//[@id="commit-url"]`

### Get all links
`//[@href]`

### Get all toolnames (may have duplicates)
`//@title`

# Querying the database

The current database contains a lot of interesting data, and below will be some examples of queries that will extract some of this data. The date is an example and may be replaced with dates you intend to look between.
@@ -44,6 +44,19 @@ def patches_query(mongoDB, inspectorJson):
return mongoDB.patches.find({"buildId" : inspectorJson['buildId']})


"""
We will be parsing html, so we need to change the < and > and & icons
"""
def replace_spec_chars(string):
string = string.replace("&", "&amp;")
string = string.replace("<", "&lt;")
string = string.replace(">", "&gt;")
string = string.replace("\"", "&quot;")
string = string.replace("\'", "&apos;")
return string



"""
Query the inspector once and return all documents
"""
@@ -64,18 +77,17 @@ def inspector_query(mongoDB):
elif(dateFrom == None and dateTo != None):
inspectorFilter = {"buildFinishedDate" : { "$lt" : dateTo}, "status" : "PATCHED"}

return mongoDB.inspector.find(inspectorFilter)
return mongoDB.inspector.find(inspectorFilter).batch_size(50)

"""
Build the string that will ultimately be written to the txt-file
patchDocs - all docs with the same buildId, so that we gather all patches
diffs - a string of the different diffse
"""
def txt_builder(patchDocs, inspectorJson):
def file_builder(patchDocs, inspectorJson):

global driver

buildURL = inspectorJson['travisURL']

driver.get(buildURL)
@@ -84,33 +96,32 @@ def txt_builder(patchDocs, inspectorJson):
commitURL = links[0].get_attribute("href")
else:
return None

diffs = []
for json in patchDocs:
diffs.append(patch_diffs(json))

diffString = "\n\n\n".join(diffs)
# Where we do have a commit url we build the html file
f = open(str(inspectorJson['buildId']) + ".html", "w")

return "\n".join([buildURL, commitURL, diffString]);

# Write commit and travis url
f.write("<html>\n<body>\n")
f.write("<p><a href=\"" + buildURL + "\" id=\"travis-url\">" + buildURL + "</a></p>\n")
f.write("<p><a href=\"" + commitURL + "\" id=\"commit-url\">" + commitURL + "</a></p>\n")



"""
Since we can have multiple patches we will have to handle these one by one as well
Will have to look at tools as well
"""
def patch_diffs(patchJson):
patchTool = patchJson['toolname']
patchDiff = patchJson['diff']

ret = patchTool + " found a patch with the following diff :\n\n" + patchDiff
print(ret)

return ret
index = 0
for json in patchDocs:
diff = json['diff']
tool = json ['toolname']
diff = replace_spec_chars(diff)
if diff != None and diff != "" and isinstance(diff, str) and tool != None:
f.write("<pre>" + tool +
"<code id=\" " + str(index) + "\" class=\"patch\" title=\"" + tool + "\">\n"
+ diff +
"</code></pre>\n")
index += 1

f.write("</body>\n</html>\n")
f.close()
return 0



"""
Fetch info and write a file for each build found
"""
@@ -124,14 +135,11 @@ def main():

for inspectorJson in inspector_query(mongoDB):
patchesDocs = patches_query(mongoDB, inspectorJson)
txt = txt_builder(patchesDocs, inspectorJson)
if(txt != None):
f = open(str(inspectorJson['buildId']) + ".txt", "w")
f.write(txt)
f.close();
file_builder(patchesDocs, inspectorJson)
print(inspectorJson['buildId'])


# Start a webdriver to make sure we can fetch the correct url
driver = webdriver.Firefox()
driver.implicitly_wait(10)
driver.implicitly_wait(5)
main()

0 comments on commit 4fc0670

Please sign in to comment.
You can’t perform that action at this time.