Skip to content

Commit

Permalink
Merge 1c40e17 into 40492db
Browse files Browse the repository at this point in the history
  • Loading branch information
@dianaclarke
dianaclarke committed Apr 26, 2021
2 parents 40492db + 1c40e17 commit b67af11
Show file tree
Hide file tree
Showing 3 changed files with 256 additions and 0 deletions.
1 change: 1 addition & 0 deletions conbench/tests/migrations/test_854c3ba5abd6_migrate_cases.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,7 @@ def test_upgrade():
assert summary_1.case_id != summary_2.case_id
assert summary_3.case_id != summary_4.case_id

# do migration
alembic_config = Config(config_path)
command.stamp(alembic_config, "991493b6406a")
command.upgrade(alembic_config, "854c3ba5abd6")
Expand Down
206 changes: 206 additions & 0 deletions conbench/tests/migrations/test_de31ab708b6c_backfill_context.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
import os

from alembic import command
from alembic.config import Config

from ...db import Session
from ...entities.run import Run
from ...entities.summary import Summary


this_dir = os.path.abspath(os.path.dirname(__file__))
config_path = os.path.join(this_dir, "../../../alembic.ini")


VALID_PAYLOAD_1 = {
"context": {
"arrow_compiler_flags": "-fPIC -arch x86_64 -arch x86_64 -std=c++11 -Qunused-arguments -fcolor-diagnostics -O3 -DNDEBUG",
"arrow_compiler_id": "AppleClang",
"arrow_compiler_version": "11.0.0.11000033",
"arrow_git_revision": "02addad336ba19a654f9c857ede546331be7b631",
"arrow_version": "2.0.0",
"benchmark_language_version": "Python 3.8.5",
"benchmark_language": "Python",
},
"run": {
"commit": "02addad336ba19a654f9c857ede546331be7b631",
"repository": "https://github.com/apache/arrow",
},
"machine_info": {
"architecture_name": "x86_64",
"cpu_l1d_cache_bytes": "32768",
"cpu_l1i_cache_bytes": "32768",
"cpu_l2_cache_bytes": "262144",
"cpu_l3_cache_bytes": "4194304",
"cpu_core_count": "2",
"cpu_frequency_max_hz": "3500000000",
"cpu_model_name": "Intel(R) Core(TM) i7-7567U CPU @ 3.50GHz",
"cpu_thread_count": "4",
"kernel_name": "19.6.0",
"memory_bytes": "17179869184",
"name": "diana",
"os_name": "macOS",
"os_version": "10.15.7",
},
"stats": {
"batch_id": "1000",
"run_id": "10000",
"run_name": "pull request: 9564",
"data": [
"0.099094",
"0.037129",
"0.036381",
"0.148896",
"0.008104",
"0.005496",
"0.009871",
"0.006008",
"0.007978",
"0.004733",
],
"times": [
"0.099094",
"0.037129",
"0.036381",
"0.148896",
"0.008104",
"0.005496",
"0.009871",
"0.006008",
"0.007978",
"0.004733",
],
"unit": "s",
"time_unit": "s",
"iqr": "0.030442",
"iterations": 10,
"max": "0.148896",
"mean": "0.036369",
"median": "0.008988",
"min": "0.004733",
"q1": "0.006500",
"q3": "0.036942",
"stdev": "0.049194",
"timestamp": "2020-11-25T21:02:42.706806+00:00",
},
"tags": {
"compression": "snappy",
"cpu_count": 2,
"dataset": "nyctaxi_sample",
"file_type": "parquet",
"input_type": "arrow",
"name": "file-write",
},
}

VALID_PAYLOAD_2 = {
"context": {
"arrow_compiler_flags": "-fPIC -arch x86_64 -arch x86_64 -std=c++11 -Qunused-arguments -fcolor-diagnostics -O3 -DNDEBUG",
"arrow_compiler_id": "AppleClang",
"arrow_compiler_version": "11.0.0.11000033",
"arrow_git_revision": "02addad336ba19a654f9c857ede546331be7b631",
"arrow_version": "2.0.0",
"benchmark_language_version": "Python 3.8.5",
"benchmark_language": "Python",
},
"run": {
"commit": "02addad336ba19a654f9c857ede546331be7b631",
"repository": "https://github.com/apache/arrow",
},
"machine_info": {
"architecture_name": "x86_64",
"cpu_l1d_cache_bytes": "32768",
"cpu_l1i_cache_bytes": "32768",
"cpu_l2_cache_bytes": "262144",
"cpu_l3_cache_bytes": "4194304",
"cpu_core_count": "2",
"cpu_frequency_max_hz": "3500000000",
"cpu_model_name": "Intel(R) Core(TM) i7-7567U CPU @ 3.50GHz",
"cpu_thread_count": "4",
"kernel_name": "19.6.0",
"memory_bytes": "17179869184",
"name": "diana",
"os_name": "macOS",
"os_version": "10.15.7",
},
"stats": {
"batch_id": "2000",
"run_id": "20000",
"run_name": "pull request: 9564",
"data": [
"0.099094",
"0.037129",
"0.036381",
"0.148896",
"0.008104",
"0.005496",
"0.009871",
"0.006008",
"0.007978",
"0.004733",
],
"times": [
"0.099094",
"0.037129",
"0.036381",
"0.148896",
"0.008104",
"0.005496",
"0.009871",
"0.006008",
"0.007978",
"0.004733",
],
"unit": "s",
"time_unit": "s",
"iqr": "0.030442",
"iterations": 10,
"max": "0.148896",
"mean": "0.036369",
"median": "0.008988",
"min": "0.004733",
"q1": "0.006500",
"q3": "0.036942",
"stdev": "0.049194",
"timestamp": "2020-11-25T21:02:42.706806+00:00",
},
"tags": {
"compression": "snappy",
"cpu_count": 2,
"dataset": "nyctaxi_sample",
"file_type": "parquet",
"input_type": "arrow",
"name": "file-write",
},
}


def test_upgrade():
summary_1 = Summary.create(VALID_PAYLOAD_1)
summary_2 = Summary.create(VALID_PAYLOAD_2)
assert summary_1.run_id == "10000"
assert summary_2.run_id == "20000"
assert summary_1.context_id is not None
assert summary_1.context_id == summary_2.context_id

run_1 = Run.get(summary_1.run_id)
run_2 = Run.get(summary_2.run_id)
assert run_1.context_id is not None
assert run_1.context_id == run_2.context_id

# assert before migration
run_1.context_id = None
run_1.save()
Session.refresh(run_1)
assert run_1.context_id is None

# do migration
alembic_config = Config(config_path)
command.stamp(alembic_config, "52b6915e289a")
command.upgrade(alembic_config, "de31ab708b6c")

# assert after migration
Session.refresh(run_1)
Session.refresh(run_2)
assert run_1.context_id is not None
assert run_1.context_id == run_2.context_id
49 changes: 49 additions & 0 deletions migrations/versions/de31ab708b6c_backfill_context.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""backfill context
Revision ID: de31ab708b6c
Revises: 52b6915e289a
Create Date: 2021-04-26 13:10:55.281793
"""
from alembic import op

from conbench.entities.run import Run
from conbench.entities.summary import Summary


# revision identifiers, used by Alembic.
revision = "de31ab708b6c"
down_revision = "52b6915e289a"
branch_labels = None
depends_on = None


def upgrade():
run_table = Run.__table__
summary_table = Summary.__table__
connection = op.get_bind()

runs = connection.execute(
run_table.select().where(run_table.c.context_id == None) # noqa
)
for run in runs:
benchmarks = list(
connection.execute(
summary_table.select()
.where(
summary_table.c.run_id == run.id,
)
.limit(1)
)
)
if benchmarks:
benchmark = benchmarks[0]
connection.execute(
run_table.update()
.where(run_table.c.id == run.id)
.values(context_id=benchmark.context_id)
)


def downgrade():
pass

0 comments on commit b67af11

Please sign in to comment.