Skip to content

Commit

Permalink
addressing comments
Browse files Browse the repository at this point in the history
  • Loading branch information
jortiz16 committed Feb 12, 2017
1 parent 6ccedfd commit 0d07b7b
Show file tree
Hide file tree
Showing 10 changed files with 196 additions and 147 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,24 @@
*
*/
public class PerfEnforceQueryMetadataEncoding {
/** The query id. */
public int id;
/** The query's SLA. */
public double slaRuntime;
/** The query's sql text */
public String queryText;

/**
* Constructor for the PerfEnforceQueryMetadataEncoding
*/
public PerfEnforceQueryMetadataEncoding() {}

/**
* Constructor for the PerfEnforceQueryMetadataEncoding
* @param id the identifier for the query
* @param slaRuntime the SLA runtime of the query
* @param queryText the string representation of the query
*/
@JsonCreator
public PerfEnforceQueryMetadataEncoding(
@JsonProperty(value = "id", required = true) final int id,
Expand All @@ -27,10 +39,18 @@ public PerfEnforceQueryMetadataEncoding(
this.queryText = queryText;
}

/**
* Returns the SLA of the query
* @return the SLA of the query
*/
public double getSLA() {
return slaRuntime;
}

/**
* Returns the string representation of the query
* @return the string representation of the query
*/
public String getQueryText() {
return queryText;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,40 @@

import java.util.List;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

/**
* This class encodes statistical information about a table from the user's schema.
*/
public class PerfEnforceStatisticsEncoding {
@Required public String table_name;
@Required public long table_size;
@Required public String selectivity_predicate_001;
@Required public String selectivity_predicate_01;
@Required public String selectivity_predicate_1;

public PerfEnforceStatisticsEncoding(
final String table_name, final long table_size, final List<String> selectivityList) {
this(
table_name,
table_size,
selectivityList.get(0),
selectivityList.get(1),
selectivityList.get(2));
}
/** The relation name */
public String table_name;
/** The relation size */
public long table_size;
/** The column value to obtain .001% of the data */
public String selectivity_predicate_001;
/** The column value to obtain .01% of the data */
public String selectivity_predicate_01;
/** The column value to obtain 1% of the data */
public String selectivity_predicate_1;

/**
* Constructor for the PerfEnforceStatisticsEncoding encoding class.
* @param table_name the name of the relation
* @param table_size the size of the relation
* @param selectivityList an array holding column values to obtain .001%, .01% and 1% of the relation
*/
@JsonCreator
public PerfEnforceStatisticsEncoding(
final String table_name,
final long table_size,
final String selectivity_predicate_001,
final String selectivity_predicate_01,
final String selectivity_predicate_1) {
@JsonProperty(value = "table_name", required = true) final String table_name,
@JsonProperty(value = "table_size", required = true) final long table_size,
@JsonProperty(value = "selectivityList", required = true)
final List<String> selectivityList) {
this.table_name = table_name;
this.table_size = table_size;
this.selectivity_predicate_001 = selectivity_predicate_001;
this.selectivity_predicate_01 = selectivity_predicate_01;
this.selectivity_predicate_1 = selectivity_predicate_1;
this.selectivity_predicate_001 = selectivityList.get(0);
this.selectivity_predicate_01 = selectivityList.get(1);
this.selectivity_predicate_1 = selectivityList.get(2);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,19 @@
* This class returns an encoding describing a table's schema.
*/
public class PerfEnforceTableEncoding {
/** The relationkey of the relation */
public RelationKey relationKey;
/** The type of the relation, which can be either "Dimension" or "fact". */
public String type;
/** The S3 source of the relation. */
public AmazonS3Source source;
/** The schema of the relation. */
public Schema schema;
/** The delimiter on the source file of the relation. */
public Character delimiter;
/** The indexes of the primary keys. Can support candidate keys. */
public Set<Integer> keys;

/** The indexes of foreign keys for the dimension tables. These keys correspond to indexes in the fact table. */
public Set<Integer> corresponding_fact_key;

/**
Expand Down
2 changes: 1 addition & 1 deletion src/edu/washington/escience/myria/parallel/Server.java
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ QueryExecutionMode getExecutionMode() {
*
* @param masterHost hostname of the master
* @param masterPort RPC port of the master
* @param catalogPath absolute path of the directory containing the master catalog files
* @param instancePath absolute path of the directory containing the master catalog files
* @param databaseSystem name of the storage DB system
* @param connectTimeoutMillis connect timeout for worker IPC
* @param sendBufferSize send buffer size in bytes for worker IPC
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@
import java.io.IOException;
import java.nio.file.Path;

import org.apache.commons.io.IOUtils;

/**
* A wrapper for the PSLAManager executable
*/
public class PSLAManagerWrapper {
final Path PSLAManagerExePath;

/**
* The constructor for the PSLAManagerWrapper class.
* Initially sets up the path pointing to the executable for PSLAManager.
*/
public PSLAManagerWrapper() {
PSLAManagerExePath =
PerfEnforceDriver.configurationPath
Expand All @@ -22,6 +24,10 @@ public PSLAManagerWrapper() {
.resolve("PSLAManager.exe");
}

/**
* Runs the PSLAManager with the "-q" flag. It generates a set of queries based on the user's dataset.
* @throws Exception if there is an error running the PSLAManager process
*/
public void generateQueries() throws Exception {
try {

Expand All @@ -34,14 +40,16 @@ public void generateQueries() throws Exception {
+ PerfEnforceDriver.configurationPath.resolve("PSLAGeneration").toString()
+ " -q");

System.out.println(IOUtils.toString(p.getErrorStream()));

p.waitFor();
} catch (IOException e) {
throw e;
}
}

/**
* Runs the PSLAManager with the "-p" flag. It generates the resulting PSLA.
* @throws Exception if there is an error running the PSLAManager process
*/
public void generatePSLA() throws Exception {
try {
Process p =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ public class PerfEnforceDataPreparation {
protected static final org.slf4j.Logger LOGGER =
LoggerFactory.getLogger(PerfEnforceDataPreparation.class);

/**
* Constructs the PerfEnforceDataPreparation class
* @param server the current instance of Server
*/
public PerfEnforceDataPreparation(final Server server) {
this.server = server;
}
Expand All @@ -64,6 +68,8 @@ public PerfEnforceDataPreparation(final Server server) {
* Ingesting the fact table in a parallel sequence
*
* @param factTableDesc contains information about the fact table to ingest
* @return a hashmap with the relationkeys for all versions of the fact table
* @throws PerfEnforceException if there is an error ingesting the fact table
*/
public HashMap<Integer, RelationKey> ingestFact(final PerfEnforceTableEncoding factTableDesc)
throws PerfEnforceException {
Expand Down Expand Up @@ -185,6 +191,7 @@ public HashMap<Integer, RelationKey> ingestFact(final PerfEnforceTableEncoding f
* Ingesting dimension tables for broadcasting
*
* @param dimTableDesc holds information about the dimension table to ingest
* @throws PerfEnforceException if there is an error ingesting dimension tables
*/
public void ingestDimension(final PerfEnforceTableEncoding dimTableDesc)
throws PerfEnforceException {
Expand Down Expand Up @@ -299,6 +306,7 @@ public void postgresStatsAnalyzeTable(

/**
* Collects statistical information about each table
* @throws PerfEnforceException if there is an error collecting data statistics
*/
public void collectSelectivities() throws PerfEnforceException {
try {
Expand Down Expand Up @@ -340,11 +348,11 @@ public void collectSelectivities() throws PerfEnforceException {
}
}

PrintWriter statsObjectWriter =
new PrintWriter(new FileOutputStream(new File(statsWorkerPath.toString())));
ObjectMapper mapper = new ObjectMapper();
mapper.writeValue(statsObjectWriter, statsEncodingList);
statsObjectWriter.close();
try (PrintWriter statsObjectWriter =
new PrintWriter(new FileOutputStream(new File(statsWorkerPath.toString())))) {
ObjectMapper mapper = new ObjectMapper();
mapper.writeValue(statsObjectWriter, statsEncodingList);
}
}
} catch (Exception e) {
throw new PerfEnforceException("Error collecting table statistics");
Expand All @@ -360,6 +368,8 @@ public void collectSelectivities() throws PerfEnforceException {
* @param type the type of the table -- can be either "fact" or "dimension"
* @param keys the primary keys of the relation
* @param schema the schema of the relation
* @return returns the statistics metadata for a relation
* @throws PerfEnforceException if there is an error computing the statistics
*/
public PerfEnforceStatisticsEncoding runTableRanking(
final RelationKey relationKey,
Expand Down Expand Up @@ -409,8 +419,9 @@ public PerfEnforceStatisticsEncoding runTableRanking(

/**
* This method collects features from each query generated by PSLAManager
* @throws PerfEnforceException if there is an error collecting features from the generated queries
*/
public void collectFeaturesFromGeneratedQueries() throws Exception {
public void collectFeaturesFromGeneratedQueries() throws PerfEnforceException {
for (Integer config : PerfEnforceDriver.configurations) {
Path workerPath =
PerfEnforceDriver.configurationPath
Expand All @@ -419,34 +430,35 @@ public void collectFeaturesFromGeneratedQueries() throws Exception {
String currentLine = "";

try {
PrintWriter featureWriter =
new PrintWriter(workerPath.resolve("TESTING.arff").toString(), "UTF-8");

featureWriter.write("@relation testing \n");
featureWriter.write("@attribute numberTables numeric \n");
featureWriter.write("@attribute postgesEstCostMin numeric \n");
featureWriter.write("@attribute postgesEstCostMax numeric \n");
featureWriter.write("@attribute postgesEstNumRows numeric \n");
featureWriter.write("@attribute postgesEstWidth numeric \n");
featureWriter.write("@attribute numberOfWorkers numeric \n");
featureWriter.write("@attribute realTime numeric \n");
featureWriter.write("\n");
featureWriter.write("@data \n");

BufferedReader br =
new BufferedReader(
new FileReader(workerPath.resolve("SQLQueries-Generated.txt").toString()));
while ((currentLine = br.readLine()) != null) {
currentLine =
currentLine.replace(
factTableDescription.relationKey.getRelationName(),
factTableRelationMapper.get(config).getRelationName());

String features = PerfEnforceUtils.getMaxFeature(server, currentLine, config);
featureWriter.write(features + "\n");
try (PrintWriter featureWriter =
new PrintWriter(workerPath.resolve("TESTING.arff").toString(), "UTF-8")) {

featureWriter.write("@relation testing \n");
featureWriter.write("@attribute numberTables numeric \n");
featureWriter.write("@attribute postgesEstCostMin numeric \n");
featureWriter.write("@attribute postgesEstCostMax numeric \n");
featureWriter.write("@attribute postgesEstNumRows numeric \n");
featureWriter.write("@attribute postgesEstWidth numeric \n");
featureWriter.write("@attribute numberOfWorkers numeric \n");
featureWriter.write("@attribute realTime numeric \n");
featureWriter.write("\n");
featureWriter.write("@data \n");

try (BufferedReader br =
new BufferedReader(
new FileReader(workerPath.resolve("SQLQueries-Generated.txt").toString()))) {
while ((currentLine = br.readLine()) != null) {
currentLine =
currentLine.replace(
factTableDescription.relationKey.getRelationName(),
factTableRelationMapper.get(config).getRelationName());

String features = PerfEnforceUtils.getMaxFeature(server, currentLine, config);
featureWriter.write(features + "\n");
}
}
}
featureWriter.close();
br.close();

} catch (Exception e) {
throw new PerfEnforceException("Error creating table features");
}
Expand Down

0 comments on commit 0d07b7b

Please sign in to comment.