From 499a7907a6e9a4f8ae803201a16e276537d00e8f Mon Sep 17 00:00:00 2001 From: Rohan Yadav Date: Wed, 24 Mar 2021 14:38:59 -0400 Subject: [PATCH] numpy,taco: updates to run full set of suitesparse benchmarks --- numpy/ufuncs.py | 31 +- scripts/suitesparse_runner.sh | 35 + scripts/taco_bench_aggregator.py | 12 +- taco/ufuncs.cpp | 34 +- valid-suitesparse.txt | 1672 ++++++++++++++++++++++++++++++ 5 files changed, 1757 insertions(+), 27 deletions(-) create mode 100755 scripts/suitesparse_runner.sh create mode 100644 valid-suitesparse.txt diff --git a/numpy/ufuncs.py b/numpy/ufuncs.py index 24888ed..57a81d3 100644 --- a/numpy/ufuncs.py +++ b/numpy/ufuncs.py @@ -92,14 +92,35 @@ def bench(): def ufunc_bench_key(tensorName, funcName): return tensorName + "-" + funcName + "-numpy" +# UfuncInputCache attempts to avoid reading the same tensor from disk multiple +# times in a benchmark run. +class UfuncInputCache: + def __init__(self): + self.lastLoaded = None + self.lastName = None + self.tensor = None + self.other = None + + def load(self, tensor, suiteSparse): + if self.lastName == str(tensor): + return self.tensor, self.other + else: + if suiteSparse: + self.lastLoaded = tensor.load(PydataMatrixMarketTensorLoader()) + else: + self.lastLoaded = tensor.load() + self.lastName = str(tensor) + self.tensor = safeCastPydataTensorToInts(self.lastLoaded) + self.other = PydataTensorShifter().shiftLastMode(self.tensor) + return self.tensor, self.other +inputCache = UfuncInputCache() + # Run benchmarks against the FROSTT collection. FROSTTTensors = TensorCollectionFROSTT() @pytest.mark.parametrize("tensor", FROSTTTensors.getTensors()) @pytest.mark.parametrize("ufunc", [numpy.logical_xor, numpy.ldexp, numpy.right_shift]) def bench_pydata_frostt_ufunc_sparse(tacoBench, tensor, ufunc): - frTensor = safeCastPydataTensorToInts(tensor.load()) - shifter = PydataTensorShifter() - other = shifter.shiftLastMode(frTensor) + frTensor, other = inputCache.load(tensor, False) def bench(): c = ufunc(frTensor, other) return c @@ -118,9 +139,7 @@ def bench(): @pytest.mark.parametrize("ufunc", [numpy.logical_xor, numpy.ldexp, numpy.right_shift]) def bench_pydata_suitesparse_ufunc_sparse(tacoBench, ufunc): tensor = SuiteSparseTensor(os.getenv('SUITESPARSE_TENSOR_PATH')) - ssTensor = safeCastPydataTensorToInts(tensor.load(PydataMatrixMarketTensorLoader())) - shifter = PydataTensorShifter() - other = shifter.shiftLastMode(ssTensor) + ssTensor, other = inputCache.load(tensor, True) def bench(): c = ufunc(ssTensor, other) return c diff --git a/scripts/suitesparse_runner.sh b/scripts/suitesparse_runner.sh new file mode 100755 index 0000000..98b4184 --- /dev/null +++ b/scripts/suitesparse_runner.sh @@ -0,0 +1,35 @@ +#!/bin/bash +#SBATCH -N 1 +#SBATCH --mem 120000 +#SBATCH -p lanka-v3 +#SBATCH --exclusive + +set -u + +source venv/bin/activate + +sspath=data/suitesparse +# out=suitesparse-ufunc-bench/taco +out=suitesparse-ufunc-bench/numpy + +mkdir -p "$out" + +while read line; do + matrix="$sspath/$line/$line.mtx" + # csvout="$out/result-$line.csv" + # LANKA=ON SUITESPARSE_TENSOR_PATH="$matrix" TACO_OUT="$csvout" make -j8 taco-bench BENCHES="bench_suitesparse_ufunc" + jsonout="$out/result-$line.json" + LANKA=ON SUITESPARSE_TENSOR_PATH="$matrix" NUMPY_JSON="$jsonout" make python-bench BENCHES="numpy/ufuncs.py::bench_pydata_suitesparse_ufunc_sparse" +done <$1 + +# for path in $sspath/*; do +# if [ ! -d $path ]; then +# continue +# fi +# name="$(cut -d'/' -f3 <<< "$path")" +# matrix="$path/$name.mtx" +# +# csvout="$out/result-$name.csv" +# +# LANKA=ON SUITESPARSE_TENSOR_PATH="$matrix" TACO_OUT="$csvout" make -j8 taco-bench BENCHES="bench_suitesparse_ufunc" +# done diff --git a/scripts/taco_bench_aggregator.py b/scripts/taco_bench_aggregator.py index 07d48c9..89b20d9 100644 --- a/scripts/taco_bench_aggregator.py +++ b/scripts/taco_bench_aggregator.py @@ -20,8 +20,8 @@ def aggregateTacoBenches(folder, outfile, labelSet=None): with open(fname, 'r') as f: # Discard the first 10 lines. This corresponds to the # google-benchmark generated header. - for i in range(0, 10): - f.readline() + # for i in range(0, 10): + # f.readline() # Open the rest of the file as a CSV. reader = csv.reader(f) # Attempt to read the header from CSV. If this fails, @@ -33,14 +33,14 @@ def aggregateTacoBenches(folder, outfile, labelSet=None): continue # Find the column that contains label. We're going to skip # entries that have a skip marker in the label. - labelIdx = header.index("label") + # labelIdx = header.index("label", 0) if first: writer.writerow(header) first = False for row in reader: - if "SKIPPED" not in row[labelIdx]: - validLabels.add(row[labelIdx]) - writer.writerow(row) + # if "SKIPPED" not in row[labelIdx]: + # validLabels.add(row[labelIdx]) + writer.writerow(row) # Write out the set of valid labels. if labelSet is not None: with open(labelSet, 'w+') as validSet: diff --git a/taco/ufuncs.cpp b/taco/ufuncs.cpp index 21c981a..5788752 100644 --- a/taco/ufuncs.cpp +++ b/taco/ufuncs.cpp @@ -161,7 +161,7 @@ TACO_BENCH_ARGS(bench_ufunc_sparse, rightShift_0.01, 0.01, ">>")->Apply(applyBen // time from caching these inputs. struct UfuncInputCache { template - std::pair, taco::Tensor> getUfuncInput(std::string path, U format) { + std::pair, taco::Tensor> getUfuncInput(std::string path, U format, bool countNNZ = false) { // See if the paths match. if (this->lastPath == path) { // TODO (rohany): Not worrying about whether the format was the same as what was asked for. @@ -175,6 +175,12 @@ struct UfuncInputCache { this->lastPath = path; this->inputTensor = castToType("A", this->lastLoaded); this->otherTensor = shiftLastMode("B", this->inputTensor); + if (countNNZ) { + this->nnz = 0; + for (auto& it : iterate(this->inputTensor)) { + this->nnz++; + } + } return std::make_pair(this->inputTensor, this->otherTensor); } @@ -183,6 +189,7 @@ struct UfuncInputCache { taco::Tensor inputTensor; taco::Tensor otherTensor; + int64_t nnz; }; UfuncInputCache inputCache; @@ -295,12 +302,14 @@ static void bench_suitesparse_ufunc(benchmark::State& state, Func op) { // Counters must be present in every run to get reported to the CSV. state.counters["dimx"] = 0; state.counters["dimy"] = 0; - if (ssTensors.tensors.size() == 0) { + state.counters["nnz"] = 0; + + auto tensorPath = getEnvVar("SUITESPARSE_TENSOR_PATH"); + if (tensorPath == "") { state.error_occurred(); return; } - int tensorIdx = state.range(0); - auto tensorPath = ssTensors.tensors[tensorIdx]; + auto pathSplit = taco::util::split(tensorPath, "/"); auto filename = pathSplit[pathSplit.size() - 1]; auto tensorName = taco::util::split(filename, ".")[0]; @@ -308,16 +317,17 @@ static void bench_suitesparse_ufunc(benchmark::State& state, Func op) { taco::Tensor ssTensor, other; try { - std::tie(ssTensor, other) = inputCache.getUfuncInput(tensorPath, CSR); + std::tie(ssTensor, other) = inputCache.getUfuncInput(tensorPath, CSR, true /* countNNZ */); } catch (TacoException& e) { // Counters don't show up in the generated CSV if we used SkipWithError, so // just add in the label that this run is skipped. - state.SetLabel(tensorName+"-SKIPPED-FAILED-READ"); + state.SetLabel(tensorName+"/SKIPPED-FAILED-READ"); return; } state.counters["dimx"] = ssTensor.getDimension(0); state.counters["dimy"] = ssTensor.getDimension(1); + state.counters["nnz"] = inputCache.nnz; for (auto _ : state) { state.PauseTiming(); @@ -339,12 +349,6 @@ static void bench_suitesparse_ufunc(benchmark::State& state, Func op) { } } -static void applySuiteSparse(benchmark::internal::Benchmark* b) { - for (int i = 0; i < ssTensors.tensors.size(); i++) { - b->Arg(i); - } -} - -TACO_BENCH_ARGS(bench_suitesparse_ufunc, xor, xorOp)->Apply(applySuiteSparse); -TACO_BENCH_ARGS(bench_suitesparse_ufunc, ldExp, ldExp)->Apply(applySuiteSparse); -TACO_BENCH_ARGS(bench_suitesparse_ufunc, rightShift, rightShift)->Apply(applySuiteSparse); +TACO_BENCH_ARGS(bench_suitesparse_ufunc, xor, xorOp); +TACO_BENCH_ARGS(bench_suitesparse_ufunc, ldExp, ldExp); +TACO_BENCH_ARGS(bench_suitesparse_ufunc, rightShift, rightShift); diff --git a/valid-suitesparse.txt b/valid-suitesparse.txt new file mode 100644 index 0000000..441a634 --- /dev/null +++ b/valid-suitesparse.txt @@ -0,0 +1,1672 @@ +c-18 +c-67b +Maragal_1 +mark3jac080 +fpga_dcop_37 +fpga_dcop_44 +freeFlyingRobot_7 +OPF_10000 +jan99jac120sc +boyd1 +hangGlider_1 +rajat16 +hangGlider_5 +fpga_dcop_41 +bcsstk08 +lp_bnl1 +bcsstk28 +barrier2-12 +mimo28x28_system +gen2 +ASIC_320ks +lhr01 +scsd8-2b +spal_004 +nos2 +lp_share2b +barrier2-9 +a0nsdsil +kineticBatchReactor_4 +shipsec5 +fxm2-6 +coater1 +bodyy5 +cage5 +freeFlyingRobot_16 +lhr07c +USAir97 +freeFlyingRobot_4 +scfxm1-2b +freeFlyingRobot_15 +freeFlyingRobot_12 +g7jac180sc +t2d_q4 +cryg10000 +lp_perold +onetone1 +ex21 +filter3D +bayer02 +invextr1_new +c-71 +mark3jac140sc +lp_scfxm1 +SiO2 +cavity08 +TSOPF_FS_b300_c3 +fpga_dcop_24 +bcsstk03 +ML_Laplace +OPF_6000 +lp_fit1d +bcsstm02 +ted_B_unscaled +g7jac120sc +para-9 +CoupCons3D +para-6 +g7jac160 +lp_scfxm2 +epb0 +bcsstm22 +lns_3937 +af_shell2 +g7jac010 +barrier2-2 +cdde6 +offshore +nemeth20 +extr1b +cavity19 +TSOPF_FS_b39_c7 +TSOPF_FS_b162_c1 +mhd4800b +apache1 +progas +g7jac080sc +t2dah +S40PI_n1 +raefsky5 +cavity15 +deter3 +oscil_dcop_32 +lp_wood1p +dwa512 +tumorAntiAngiogenesis_1 +bcsstm34 +bcsstm09 +cavity16 +freeFlyingRobot_9 +lp_ganges +astro-ph +FEM_3D_thermal2 +spaceShuttleEntry_2 +model4 +nmos3 +para-4 +nnc666 +adder_dcop_30 +nasa2146 +qh768 +bayer07 +lp_osa_60 +c-68 +gre_216b +bayer05 +Goodwin_010 +lowThrust_6 +t3dl_a +adder_dcop_46 +LeGresley_4908 +adder_dcop_51 +stat96v2 +rajat12 +bfwb62 +fpga_dcop_23 +stokes128 +adder_dcop_50 +car4 +TSOPF_FS_b39_c19 +fd15 +oscil_dcop_12 +str_600 +bayer04 +oscil_dcop_28 +gams30am +tp-6 +lp_scagr7 +adder_dcop_48 +msdoor +darcy003 +crystk03 +finan512 +ss +reorientation_8 +S80PI_n +cont-201 +bcsstm23 +oscil_dcop_22 +662_bus +impcol_c +dw4096 +tols4000 +tuma2 +af_shell5 +freeFlyingRobot_10 +ex23 +S80PI_n1 +xingo_afonso_itaipu +bcsstk38 +hood +reorientation_4 +lp_fit2p +TSOPF_RS_b300_c3 +test1 +g7jac060sc +fpga_dcop_14 +dynamicSoaringProblem_1 +ex24 +adder_dcop_14 +af_5_k101 +kineticBatchReactor_2 +oscil_dcop_54 +mahindas +bcsstk17 +lp_sc205 +LargeRegFile +lp_pilotnov +lhr34 +bmwcra_1 +lp_agg +dano3mip +cond-mat +ACTIVSg70K +ex22 +tumorAntiAngiogenesis_2 +tols90 +msc00726 +c-62 +lowThrust_8 +vas_stokes_1M +adder_dcop_29 +fpga_dcop_25 +memchip +g7jac140 +barrier2-11 +pores_2 +mesh3em5 +mario001 +Hook_1498 +fpga_dcop_17 +Goodwin_127 +raefsky1 +adder_dcop_23 +fv1 +watson_2 +lpi_pang +power197k +Maragal_6 +std1_Jac3 +ASIC_320k +lop163 +mark3jac040 +adder_dcop_52 +dielFilterV3real +reorientation_5 +t2dal_e +west1505 +cavity07 +spaceStation_3 +adder_dcop_45 +nemeth24 +g7jac180 +spaceStation_13 +Serena +laminar_duct3D +cavity11 +lp_standata +lp_agg2 +d_pretok +c-38 +west0156 +c-45 +s2rmq4m1 +b1_ss +kineticBatchReactor_1 +freeFlyingRobot_1 +cage15 +c-64b +SiO +atmosmodl +oscil_dcop_30 +Si10H16 +flowmeter0 +cq9 +freeFlyingRobot_5 +steam1 +lp_grow22 +scrs8-2c +dynamicSoaringProblem_5 +bcsstk35 +mcfe +lhr11c +bcsstk20 +TEM27623 +ex27 +xenon2 +fpga_dcop_12 +ncvxbqp1 +epb1 +TSOPF_RS_b678_c2 +nos7 +lp_bandm +s1rmq4m1 +adder_dcop_68 +saylr4 +freeFlyingRobot_11 +bayer06 +nemeth17 +oscil_dcop_17 +g7jac050sc +F2 +orbitRaising_2 +mimo8x8_system +nemsemm2 +nemeth16 +ncvxqp3 +lhr71 +lund_b +netscience +bcsstm20 +dw256B +shallow_water1 +c-70 +c-54 +stokes64 +fpga_dcop_06 +baxter +c-69 +TSOPF_RS_b9_c6 +kineticBatchReactor_5 +cage7 +QRpivot +jan99jac100 +watt_2 +lp_sc50a +adder_dcop_09 +G2_circuit +beause +Si5H12 +adder_dcop_58 +oscil_dcop_26 +fpga_dcop_04 +c-58 +cavity23 +init_adder1 +heart2 +adder_dcop_16 +bcsstm36 +fpga_dcop_26 +ted_A +rdb2048 +dendrimer +mesh3e1 +gre_512 +lp_scsd1 +bayer10 +oscil_dcop_11 +olm5000 +wang4 +c-40 +ncvxqp7 +msc23052 +cage12 +494_bus +poli4 +rajat29 +bips98_1142 +tomography +mhd3200b +denormal +cz5108 +trans5 +lowThrust_4 +delf +t2dah_a +adder_dcop_01 +lp_e226 +pre2 +oscil_dcop_16 +utm300 +sme3Dc +Bump_2911 +lp_pilot4 +lhr10 +c-30 +t3dh +circuit_3 +lp_osa_07 +lp_ship12l +ww_36_pmec_36 +Goodwin_095 +M10PI_n1 +Hamrle1 +bcsstk06 +adder_dcop_41 +nemswrld +gams60am +cbuckle +psse1 +aug3dcqp +adder_dcop_40 +Goodwin_040 +PFlow_742 +mesh2e1 +nemeth25 +li +rajat13 +dc2 +rw5151 +t3dh_e +ex28 +l30 +oscil_dcop_53 +laser +G3_circuit +ACTIVSg10K +meg1 +c-62ghs +lp_stocfor3 +model9 +fs_183_6 +water_tank +CurlCurl_3 +cavity24 +FullChip +kkt_power +reorientation_1 +rma10 +Kemelmacher +lowThrust_2 +Delor295K +TEM152078 +ldoor +poli +Freescale2 +co5 +mesh1em6 +qh882 +lp_ship04s +bcsstk23 +bcsstm05 +c-59 +chipcool1 +Freescale1 +sinc15 +c-35 +c-42 +TSOPF_FS_b9_c1 +adder_dcop_35 +adder_dcop_33 +sc205-2r +shermanACa +raefsky2 +bcsstm06 +nemeth01 +Zd_Jac3_db +tmt_sym +lp_finnis +lp_osa_14 +gre_216a +fv3 +c-24 +lp_25fv47 +ex31 +bcsstm12 +cdde2 +s1rmt3m1 +poli_large +Ge99H100 +oscil_dcop_34 +torsion1 +fpga_dcop_01 +bcsstm11 +cdde1 +g7jac100 +cz40948 +nxp1 +psmigr_3 +cond-mat-2003 +cz10228 +bodyy6 +degme +impcol_d +gams10am +Zd_Jac3 +scrs8-2b +adder_dcop_42 +lp_stocfor2 +fpga_dcop_40 +fpga_dcop_50 +stokes64s +fs_760_3 +imagesensor +cr42 +p010 +para-8 +spaceStation_2 +viscorocks +Hamrle3 +atmosmodj +onetone2 +ex20 +ecology2 +jan99jac060 +ns3Da +bcsstk27 +shyy161 +bcsstm10 +TSOPF_RS_b2052_c1 +nemeth06 +rail_20209 +F1 +case39 +lp_ship08l +gre_115 +t3dl_e +nemeth19 +k1_san +ACTIVSg2000 +Delor338K +tumorAntiAngiogenesis_4 +de080285 +lhr71c +lnsp_511 +lp_blend +jan99jac020sc +lp_fit2d +fpga_dcop_13 +oscil_dcop_15 +bfwb782 +lhr04c +mesh1em1 +rajat27 +rajat24 +psmigr_2 +lp_czprob +fpga_dcop_31 +af_shell6 +wang1 +piston +rajat28 +spaceStation_10 +adder_dcop_49 +b2_ss +benzene +adder_dcop_06 +c-20 +bp_1600 +Goodwin_023 +lowThrust_13 +west0067 +nasa1824 +bips98_1450 +adder_dcop_38 +crankseg_1 +av41092 +fxm2-16 +radfr1 +bcsstk13 +nos5 +Cube_Coup_dt0 +lhr17c +ex2 +cavity01 +rajat23 +m3plates +bcsstk02 +GaAsH6 +fs_541_2 +venkat50 +Si2 +bcsstk39 +oscil_dcop_48 +oscil_dcop_24 +ex18 +ncvxqp5 +fpga_dcop_30 +wm3 +str_200 +olm100 +lpi_cplex2 +nemeth09 +epb3 +c-36 +rajat21 +small +dixmaanl +ex10hs +Geo_1438 +fpga_dcop_21 +webbase-1M +coupled +lp_ship12s +gyro +lowThrust_10 +spaceStation_11 +fs_680_2 +illc1033 +Maragal_5 +cavity17 +Rucci1 +oscil_dcop_09 +lpi_chemcom +well1033 +cage4 +bp_800 +orsirr_2 +rajat17 +lowThrust_7 +LeGresley_87936 +lp_afiro +sit100 +qiulp +nemsemm1 +chem_master1 +deter0 +cavity26 +adder_dcop_54 +utm1700b +thermomech_TK +stat96v3 +heart1 +HV15R +dynamicSoaringProblem_2 +Kuu +Sieber +filter2D +fs_680_1 +c-64 +thermal1 +lp_tuff +ecology1 +c-61 +deter6 +aft01 +nopss_11k +heart3 +adder_dcop_34 +ecl32 +TSOPF_FS_b300 +ct20stif +matrix-new_3 +refine +adder_dcop_27 +bcsstk26 +adder_dcop_20 +oscil_dcop_29 +af_4_k101 +nemeth05 +rdb450 +lp_modszk1 +dynamicSoaringProblem_6 +msc01440 +stormg2-27 +bcsstk34 +orsreg_1 +lp_stair +model6 +impcol_e +gen4 +hydr1 +bcsstm13 +model5 +d_dyn1 +ckt11752_dc_1 +adder_dcop_04 +adder_dcop_28 +ex37 +lp_share1b +Ga41As41H72 +airfoil_2d +psse2 +ted_B +tumorAntiAngiogenesis_8 +Kaufhold +co9 +largebasis +fpga_dcop_10 +circuit204 +copter2 +bips07_1693 +chipcool0 +oscil_dcop_44 +tols1090 +rotor2 +scircuit +lp_kb2 +nemspmm1 +bp_0 +adder_dcop_12 +lp_sc105 +lowThrust_1 +Maragal_4 +p0291 +nemeth15 +bwm2000 +thermal +scrs8-2r +tumorAntiAngiogenesis_6 +oscil_dcop_27 +adder_dcop_66 +lp_pilot +mesh2em5 +fs_183_1 +oscil_dcop_08 +oscil_trans_01 +atmosmodd +rajat30 +HTC_336_9129 +freeFlyingRobot_13 +Delor64K +dynamicSoaringProblem_4 +inlet +sme3Db +fpga_dcop_51 +LeGresley_2508 +shipsec1 +lnsp_131 +torso2 +nemeth04 +sinc12 +fpga_dcop_16 +nasasrb +oscil_dcop_49 +af_shell9 +spaceShuttleEntry_4 +TSOPF_FS_b162_c3 +lpi_pilot4i +olafu +bcsstk04 +lhr17 +l9 +lp_ship08s +lhr14 +str_0 +rdb200 +oscil_dcop_45 +extr1 +Ill_Stokes +ex10 +scagr7-2r +gams30a +fpga_dcop_20 +cavity06 +ex1 +shermanACd +freeFlyingRobot_2 +rajat26 +r05 +Zd_Jac6_db +add20 +lp_cre_a +af_2_k101 +nl +mark3jac060 +spaceStation_8 +cont-300 +bcsstk15 +rdb968 +adder_trans_02 +Raj1 +cavity20 +spaceShuttleEntry_3 +freeFlyingRobot_3 +spaceShuttleEntry_1 +eurqsa +mbeacxc +tomographic1 +nlpkkt120 +SiNa +t3dh_a +BenElechi1 +thread +bcsstk16 +rajat11 +sherman3 +lp_vtp_base +ex13 +lowThrust_3 +g7jac040sc +twotone +windtunnel_evap2d +lp_scrs8 +cant +smt +Flan_1565 +adder_dcop_19 +M80PI_n +spaceStation_4 +cvxbqp1 +gas11 +dw8192 +lowThrust_5 +hangGlider_3 +fpga_dcop_38 +cavity12 +bcsstk12 +mult_dcop_02 +Maragal_8 +nlpkkt80 +steam3 +Baumann +thermomech_TC +nos3 +circuit_1 +sts4098 +Hardesty3 +TSC_OPF_300 +cage11 +c-48 +tols340 +lung1 +rotor1 +photogrammetry +model7 +c-34 +adder_dcop_67 +jan99jac100sc +fxm3_16 +cdde4 +nemspmm2 +beaflw +epb2 +mac_econ_fwd500 +plddb +Goodwin_071 +ibm_matrix_2 +bcsstk18 +goddardRocketProblem_2 +DK01R +cz20468 +gemat1 +stat96v1 +saylr3 +g7jac020sc +helm3d01 +2D_54019_highK +lp_recipe +adder_dcop_21 +bmw3_2 +cvxqp3 +lp_truss +fpga_dcop_08 +adder_dcop_53 +Maragal_2 +hangGlider_2 +rdist2 +bloweybl +fpga_dcop_39 +utm3060 +jan99jac120 +Chebyshev1 +lund_a +nv2 +nemeth21 +str_400 +lhr10c +mhda416 +barrier2-1 +crystk02 +S40PI_n +cavity03 +fpga_dcop_35 +nnc261 +SiH4 +poisson3Da +gsm_106857 +lpi_forest6 +g7jac200sc +tub100 +adder_dcop_43 +lp_ship04l +ted_A_unscaled +lhr04 +RM07R +vas_stokes_2M +oscil_dcop_39 +fs_541_3 +bcsstk09 +lp_woodw +comsol +oscil_dcop_47 +Goodwin_030 +nemeth03 +fpga_dcop_03 +c-73 +lp_maros +Hamrle2 +fd12 +mri2 +adder_dcop_62 +oscil_dcop_19 +bcsstk25 +memplus +TSOPF_RS_b162_c1 +adder_dcop_10 +spmsrtls +cage10 +de063157 +oscil_dcop_46 +M20PI_n +fs_541_1 +fpga_dcop_43 +lpi_itest6 +de063155 +mhd4800a +impcol_b +pde225 +language +lhr34c +c-57 +bcsstk05 +qh1484 +mult_dcop_01 +fpga_dcop_22 +rdb3200l +bcsstm03 +lp_standgub +dw1024 +fs_183_4 +spaceStation_7 +s3rmt3m3 +adder_dcop_56 +c-31 +TSOPF_RS_b678_c1 +cryg2500 +sherman1 +reorientation_2 +s2rmt3m1 +analytics +lpi_bgetam +adder_dcop_69 +kineticBatchReactor_3 +bayer03 +bbmat +turon_m +MISKnowledgeMap +bcsstm27 +adder_dcop_63 +shermanACb +reorientation_7 +s3dkt3m2 +LFAT5000 +adder_dcop_37 +c-47 +world +bp_1400 +bfwa62 +fpga_dcop_27 +bcsstk22 +stormG2_1000 +nemeth12 +Chem97ZtZ +fpga_dcop_02 +nemeth14 +Zd_Jac2 +fd18 +bcsstm35 +bayer08 +nemeth02 +lp_dfl001 +scagr7-2c +cage3 +lpi_qual +image_interp +xingo3012 +ex32 +mark3jac100sc +jan99jac020 +lp_beaconfd +west0655 +lp_scfxm3 +vibrobox +Dubcova3 +rajat31 +gen1 +orsirr_1 +LF10 +t2dal_bci +mixtank_new +rail_1357 +b_dyn +zed +Goodwin_013 +S20PI_n +cavity09 +apache2 +mbeaflw +c-23 +wm2 +thermomech_dK +fome11 +scagr7-2b +dtoc +zenios +crystm02 +c-19 +ASIC_680k +fpga_dcop_33 +jan99jac040sc +rajat22 +fome13 +lp_80bau3b +tumorAntiAngiogenesis_5 +2D_27628_bjtcai +deter2 +mark3jac120sc +StocF-1465 +S20PI_n1 +g7jac020 +boyd2 +gams10a +H2O +oscil_dcop_06 +photogrammetry2 +graham1 +S10PI_n +nv1 +cz308 +c-39 +lp_grow15 +west0989 +bcsstm25 +bfwb398 +olm2000 +raefsky3 +dw2048 +barrier2-4 +obstclae +TEM181302 +cq5 +bwm200 +impcol_a +stormg2-125 +ex6 +c-37 +psse0 +jan99jac080sc +transient +s3rmt3m1 +raefsky6 +c-56 +orbitRaising_3 +af_shell1 +ex11 +problem1 +lp_maros_r7 +t3dl +nemeth08 +dc3 +west0381 +deter5 +3D_51448_3D +fpga_dcop_28 +sls +lp_gfrd_pnc +c-26 +scfxm1-2r +dw256A +FA +cont1_l +trans4 +para-10 +c-21 +stat96v5 +lhr14c +wang3 +lns_131 +deter4 +TS +ASIC_100ks +fpga_dcop_46 +iiasa +scsd8-2c +m_t1 +adder_dcop_32 +mark3jac080sc +rw496 +lp_grow7 +poisson2D +oscil_dcop_31 +sherman2 +fpga_dcop_47 +gridgena +g7jac040 +ex19 +Chebyshev4 +TSOPF_FS_b300_c2 +nemeth26 +lpi_greenbea +adder_dcop_03 +oscil_dcop_36 +vanbody +spaceStation_1 +qa8fk +hydr1c +young3c +msc01050 +bcsstm21 +lp_scorpion +goddardRocketProblem_1 +cfd2 +lp_cre_d +raefsky4 +Pres_Poisson +wang2 +robot +adder_dcop_25 +mario002 +rajat15 +rdist1 +lp_scagr25 +cont11_l +dc1 +fs_760_2 +ch +adder_dcop_60 +c-27 +pivtol +nos6 +oscil_dcop_14 +bips98_606 +CurlCurl_0 +lp_bnl2 +s3rmq4m1 +qpband +lp_scsd8 +barrier2-10 +c-66b +k3plates +ship_003 +ck656 +lp_fffff800 +cep1 +spiral +3D_28984_Tetra +oscil_dcop_05 +rajat05 +add32 +arc130 +model3 +west0479 +adder_dcop_02 +bratu3d +nemeth13 +bcsstk14 +deltaX +circuit5M +orani678 +ww_vref_6405 +adder_dcop_39 +mimo46x46_system +lp_adlittle +TSOPF_FS_b9_c6 +oscil_dcop_04 +watson_1 +mark3jac120 +pwtk +M40PI_n1 +exdata_1 +cage13 +lp_bore3d +c-44 +TSOPF_RS_b162_c4 +fpga_dcop_19 +rdist3a +cz628 +ex5 +cavity21 +nemeth18 +LFAT5 +Fault_639 +stat96v4 +cavity18 +Zhao1 +adder_trans_01 +swang2 +adder_dcop_18 +tumorAntiAngiogenesis_3 +cell2 +adder_dcop_57 +c-55 +specular +adder_dcop_22 +PR02R +c-28 +msc10848 +coater2 +lp_stocfor1 +c-65 +fpga_dcop_36 +Transport +sparsine +fp +oscil_dcop_40 +adder_dcop_31 +ex40 +illc1850 +jan99jac080 +lp_d2q06c +adder_dcop_08 +bcsstk01 +gen +t2dal_a +lpi_cplex1 +d_ss +cage8 +lp_standmps +spaceStation_12 +powersim +lp_cre_b +a5esindl +TSOPF_RS_b300_c1 +ex3 +wm1 +CO +model8 +torso3 +rw136 +fpga_trans_01 +M40PI_n +d_dyn +OPF_3754 +Chebyshev3 +Si87H76 +GT01R +oscil_dcop_43 +adder_dcop_15 +descriptor_xingo6u +ck104 +lp_capri +CurlCurl_4 +majorbasis +oscil_dcop_13 +Ge87H76 +rajat18 +appu +c-51 +odepb400 +nemeth07 +ex36 +bcsstk36 +M80PI_n1 +adder_dcop_24 +Zhao2 +patents_main +fpga_dcop_05 +ex26 +hangGlider_4 +adder_dcop_59 +deter7 +c-32 +rajat19 +af_1_k101 +adder_dcop_55 +mesh_deform +ncvxqp9 +minsurfo +adder_dcop_17 +ship_001 +cz148 +linverse +c-29 +jnlbrng1 +oscil_dcop_35 +g7jac140sc +bcsstm38 +dynamicSoaringProblem_7 +consph +venkat25 +bmw7st_1 +M20PI_n1 +oscil_dcop_56 +fpga_dcop_11 +human_gene1 +c-49 +adder_dcop_47 +bodyy4 +bcsstk24 +bp_1000 +gre_343 +TSOPF_RS_b2383 +cell1 +graphics +cdde3 +g7jac120 +bcsstm24 +g7jac010sc +TSOPF_FS_b300_c1 +barrier2-3 +tmt_unsym +ABACUS_shell_ud +gemat11 +e40r0100 +mcca +Queen_4147 +nemscem +poli3 +nd24k +Pd_rhs +cz2548 +c-50 +c-big +bloweya +bcsstm08 +lpi_refinery +reorientation_3 +bcsstm04 +Emilia_923 +pde2961 +bcsstk11 +fpga_dcop_42 +crystk01 +plat1919 +gyro_k +west2021 +crystm03 +cavity14 +fpga_dcop_32 +gas_sensor +rdb200l +lnsp3937 +nnc1374 +pores_3 +ex12 +bcsstk19 +ex35 +oscil_dcop_21 +cavity10 +lp_cre_c +pdb1HYS +ex9 +igbt3 +Ga3As3H12 +zeros_nopss_13k +Hardesty2 +ML_Geer +poisson3Db +south31 +TSOPF_RS_b300_c2 +slptsk +Na5 +case9 +model10 +af_shell10 +oscil_dcop_41 +mouse_gene +viscoplastic1 +oscil_dcop_38 +orbitRaising_4 +lowThrust_9 +ifiss_mat +mhdb416 +odepa400 +JP +lhr02 +lowThrust_12 +ckt11752_tr_0 +oscil_dcop_02 +para-5 +mosfet2 +p05 +bfwa398 +c-67 +mark3jac040sc +bips07_3078 +g7jac200 +af_3_k101 +boneS01 +HTC_336_4438 +oscil_dcop_37 +oscil_dcop_52 +ck400 +nos4 +cop20k_A +std1_Jac3_db +nlpkkt160 +lp_agg3 +juba40k +fpga_dcop_29 +fpga_dcop_09 +karted +sherman5 +cage14 +GD97_b +lp_greenbea +inline_1 +msc04515 +rajat03 +Goodwin_054 +c-46 +spaceStation_6 +hep-th +ex8 +swang1 +c-22 +parabolic_fem +bcsstm07 +Lin +Pd +fpga_dcop_18 +mod2 +hcircuit +c-73b +TSOPF_RS_b2383_c1 +ge +lp_pilot87 +bcsstm37 +Ga19As19H42 +cavity02 +c-33 +Si34H36 +tuma1 +lp_sc50b +rajat04 +cavity04 +oscil_dcop_18 +ex33 +sherman4 +TSOPF_RS_b39_c30 +oscil_dcop_01 +lns_511 +fpga_dcop_45 +mhd3200a +2cubes_sphere +stomach +bcsstk37 +testbig +Cube_Coup_dt6 +freeFlyingRobot_14 +cond-mat-2005 +adder_dcop_61 +nemeth10 +deter8 +cz1268 +lp_brandy +nemeth23 +adder_dcop_26 +bayer09 +oscil_dcop_03 +lp_scsd6 +lhr11 +hvdc2 +rat +ex25 +matrix_9 +Dubcova2 +mark3jac140 +ASIC_680ks +bips07_1998 +well1850 +tumorAntiAngiogenesis_7 +fs_760_1 +marine1 +1138_bus +adder_dcop_65 +ex15 +fpga_dcop_48 +cage6 +dgreen +af_shell8 +oscil_dcop_20 +shallow_water2 +power9 +gemat12 +fs_183_3 +CurlCurl_2 +nemsafm +garon1 +bauru5727 +lp_osa_30 +bfwa782 +shyy41 +c-25 +lp_cycle +sinc18 +685_bus +nemeth22 +rajat20 +cavity22 +TSC_OPF_1047 +cavity13 +rdb800l +af_shell4 +oscil_dcop_50 +flowmeter5 +fome12 +jendrec1 +cavity25 +watt_1 +Long_Coup_dt0 +utm5940 +xenon1 +lpi_bgdbg1 +adder_dcop_07 +bcsstm19 +rdb5000 +garon2 +c-43 +orbitRaising_1 +cari +hvdc1 +bundle_adj +a2nnsnsl +mark3jac060sc +lp_lotfi +Zd_Jac2_db +plat362 +Maragal_7 +M10PI_n +shipsec8 +Zd_Jac6 +TSOPF_FS_b39_c30 +circuit_2 +olm500 +ex14 +ex7 +nd6k +bcsstk21 +std1_Jac2_db +cage9 +oscil_dcop_10 +adder_dcop_36 +rim +lpi_gosh +nd3k +lp_etamacro +scsd8-2r +fpga_dcop_34 +mbeause +ts-palko +Dubcova1 +s3dkq4m2 +af_shell3 +saylr1 +bcsstm39 +ncvxqp1 +cfd1 +radiation +adder_dcop_64 +bcircuit +Ga10As10H30 +crashbasis +c-72 +Long_Coup_dt6 +kineticBatchReactor_6 +pde900 +viscoplastic2 +dawson5 +stormg2-8 +dwb512 +FEM_3D_thermal1 +olm1000 +adder_dcop_05 +oscil_dcop_55 +Si41Ge41H72 +mark3jac020 +wathen120 +c-53 +oscil_dcop_42 +bp_400 +oscil_dcop_33 +bp_600 +west0132 +wathen100 +mark3jac100 +freeFlyingRobot_6 +rdb2048_noL +fpga_trans_02 +ulevimin +adder_dcop_44 +nd12k +adder_dcop_11 +circuit5M_dc +steam2 +rajat25 +ESOC +TSOPF_RS_b39_c7 +Muu +af_shell7 +TSOPF_FS_b162_c4 +rail_79841 +bcsstm26 +kineticBatchReactor_8 +lp_israel +iprob +venkat01 +Chebyshev2 +thermal2 +bp_1200 +kineticBatchReactor_9 +c-60 +thermomech_dM +ohne2 +gyro_m +gre_185 +bcsstk10 +torso1 +tols2000 +std1_Jac2 +blockqp1 +rdb1250 +dynamicSoaringProblem_8 +c-41 +sme3Da +human_gene2 +lung2 +freeFlyingRobot_8 +fpga_dcop_49 +oscil_dcop_57 +boneS10 +para-7 +c-52 +nemeth11 +ex4 +S10PI_n1 +g7jac060 +pores_1 +af23560 +bips07_2476 +deter1 +jan99jac060sc +cavity05 +c-63 +lp_pilot_we +LF10000 +brainpc2 +reorientation_6 +g7jac080 +vas_stokes_4M +rdb450l +fpga_dcop_07 +landmark +audikw_1 +mark3jac020sc +lhr07 +t2dah_e +mesh1e1 +bundle1 +spaceStation_9 +neos +lpi_itest2 +dielFilterV2real +jan99jac040 +goodwin +t2dal +c-66 +TSOPF_RS_b39_c19 +lpi_vol1 +tub1000 +kineticBatchReactor_7 +oilpan +large +beacxc +fs_541_4 +t2d_q9 +lp_pilot_ja +model2 +circuit_4 +ss1 +lpi_reactor +adder_dcop_13 +CurlCurl_1 +spaceStation_5 +oscil_dcop_23 +dynamicSoaringProblem_3 +hor_131 +mri1 +west0167 +fpga_dcop_15 +lp_greenbeb +crystm01 +windtunnel_evap3d +oscil_dcop_51 +fv2 +bp_200 +fs_680_3 +lowThrust_11 +lp_fit1p +qa8fm +bayer01 +helm2d03 +gre_1107 +route +rail_5177 +g7jac100sc +bcsstk07 +lpi_gran +mult_dcop_03 +oscil_dcop_07 +meg4 +oscil_dcop_25 +rdb1250l +TSOPF_RS_b162_c3 +cdde5 +ASIC_100k +af_0_k101 +model1 +west0497 +lpi_bgindy +bloweybq +plbuckle +ex29 +crankseg_2 +Goodwin_017 +rajat14 +spaceStation_14 +bone010 +Maragal_3 +olesnik0 +g7jac160sc +x104