Merge pull request #58791 from ClickHouse/better-stateful-tests

Simplify stateful tests

cmake/sanitize.cmake

@@ -82,3 +82,4 @@ if (SANITIZE_COVERAGE)
 endif()
 
 set (WITHOUT_COVERAGE_FLAGS "-fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table")
+set (WITHOUT_COVERAGE_FLAGS_LIST -fno-profile-instr-generate -fno-coverage-mapping -fno-sanitize-coverage=trace-pc-guard,pc-table)

contrib/jemalloc-cmake/CMakeLists.txt

@@ -161,6 +161,9 @@ target_include_directories(_jemalloc SYSTEM PRIVATE
 
 target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
 
+# Because our coverage callbacks call malloc, and recursive call of malloc could not work.
+target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST})
+
 if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
     target_compile_definitions(_jemalloc PRIVATE
         -DJEMALLOC_DEBUG=1

docker/test/stateful/Dockerfile

@@ -11,14 +11,6 @@ RUN apt-get update -y \
         npm \
     && apt-get clean
 
-COPY s3downloader /s3downloader
-
-ENV S3_URL="https://clickhouse-datasets.s3.amazonaws.com"
-ENV DATASETS="hits visits"
-
-# The following is already done in clickhouse/stateless-test
-# RUN npm install -g azurite
-# RUN npm install tslib
-
+COPY create.sql /
 COPY run.sh /
 CMD ["/bin/bash", "/run.sh"]

docker/test/stateful/create.sql

@@ -0,0 +1,333 @@
+ATTACH TABLE datasets.hits_v1 UUID '78ebf6a1-d987-4579-b3ec-00c1a087b1f3'
+(
+    WatchID UInt64,
+    JavaEnable UInt8,
+    Title String,
+    GoodEvent Int16,
+    EventTime DateTime,
+    EventDate Date,
+    CounterID UInt32,
+    ClientIP UInt32,
+    ClientIP6 FixedString(16),
+    RegionID UInt32,
+    UserID UInt64,
+    CounterClass Int8,
+    OS UInt8,
+    UserAgent UInt8,
+    URL String,
+    Referer String,
+    URLDomain String,
+    RefererDomain String,
+    Refresh UInt8,
+    IsRobot UInt8,
+    RefererCategories Array(UInt16),
+    URLCategories Array(UInt16),
+    URLRegions Array(UInt32),
+    RefererRegions Array(UInt32),
+    ResolutionWidth UInt16,
+    ResolutionHeight UInt16,
+    ResolutionDepth UInt8,
+    FlashMajor UInt8,
+    FlashMinor UInt8,
+    FlashMinor2 String,
+    NetMajor UInt8,
+    NetMinor UInt8,
+    UserAgentMajor UInt16,
+    UserAgentMinor FixedString(2),
+    CookieEnable UInt8,
+    JavascriptEnable UInt8,
+    IsMobile UInt8,
+    MobilePhone UInt8,
+    MobilePhoneModel String,
+    Params String,
+    IPNetworkID UInt32,
+    TraficSourceID Int8,
+    SearchEngineID UInt16,
+    SearchPhrase String,
+    AdvEngineID UInt8,
+    IsArtifical UInt8,
+    WindowClientWidth UInt16,
+    WindowClientHeight UInt16,
+    ClientTimeZone Int16,
+    ClientEventTime DateTime,
+    SilverlightVersion1 UInt8,
+    SilverlightVersion2 UInt8,
+    SilverlightVersion3 UInt32,
+    SilverlightVersion4 UInt16,
+    PageCharset String,
+    CodeVersion UInt32,
+    IsLink UInt8,
+    IsDownload UInt8,
+    IsNotBounce UInt8,
+    FUniqID UInt64,
+    HID UInt32,
+    IsOldCounter UInt8,
+    IsEvent UInt8,
+    IsParameter UInt8,
+    DontCountHits UInt8,
+    WithHash UInt8,
+    HitColor FixedString(1),
+    UTCEventTime DateTime,
+    Age UInt8,
+    Sex UInt8,
+    Income UInt8,
+    Interests UInt16,
+    Robotness UInt8,
+    GeneralInterests Array(UInt16),
+    RemoteIP UInt32,
+    RemoteIP6 FixedString(16),
+    WindowName Int32,
+    OpenerName Int32,
+    HistoryLength Int16,
+    BrowserLanguage FixedString(2),
+    BrowserCountry FixedString(2),
+    SocialNetwork String,
+    SocialAction String,
+    HTTPError UInt16,
+    SendTiming Int32,
+    DNSTiming Int32,
+    ConnectTiming Int32,
+    ResponseStartTiming Int32,
+    ResponseEndTiming Int32,
+    FetchTiming Int32,
+    RedirectTiming Int32,
+    DOMInteractiveTiming Int32,
+    DOMContentLoadedTiming Int32,
+    DOMCompleteTiming Int32,
+    LoadEventStartTiming Int32,
+    LoadEventEndTiming Int32,
+    NSToDOMContentLoadedTiming Int32,
+    FirstPaintTiming Int32,
+    RedirectCount Int8,
+    SocialSourceNetworkID UInt8,
+    SocialSourcePage String,
+    ParamPrice Int64,
+    ParamOrderID String,
+    ParamCurrency FixedString(3),
+    ParamCurrencyID UInt16,
+    GoalsReached Array(UInt32),
+    OpenstatServiceName String,
+    OpenstatCampaignID String,
+    OpenstatAdID String,
+    OpenstatSourceID String,
+    UTMSource String,
+    UTMMedium String,
+    UTMCampaign String,
+    UTMContent String,
+    UTMTerm String,
+    FromTag String,
+    HasGCLID UInt8,
+    RefererHash UInt64,
+    URLHash UInt64,
+    CLID UInt32,
+    YCLID UInt64,
+    ShareService String,
+    ShareURL String,
+    ShareTitle String,
+    "ParsedParams.Key1" Array(String),
+    "ParsedParams.Key2" Array(String),
+    "ParsedParams.Key3" Array(String),
+    "ParsedParams.Key4" Array(String),
+    "ParsedParams.Key5" Array(String),
+    "ParsedParams.ValueDouble" Array(Float64),
+    IslandID FixedString(16),
+    RequestNum UInt32,
+    RequestTry UInt8
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(EventDate)
+ORDER BY (CounterID, EventDate, intHash32(UserID))
+SAMPLE BY intHash32(UserID)
+SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G',
+         disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/'));
+
+ATTACH TABLE datasets.visits_v1 UUID '5131f834-711f-4168-98a5-968b691a104b'
+(
+    CounterID UInt32,
+    StartDate Date,
+    Sign Int8,
+    IsNew UInt8,
+    VisitID UInt64,
+    UserID UInt64,
+    StartTime DateTime,
+    Duration UInt32,
+    UTCStartTime DateTime,
+    PageViews Int32,
+    Hits Int32,
+    IsBounce UInt8,
+    Referer String,
+    StartURL String,
+    RefererDomain String,
+    StartURLDomain String,
+    EndURL String,
+    LinkURL String,
+    IsDownload UInt8,
+    TraficSourceID Int8,
+    SearchEngineID UInt16,
+    SearchPhrase String,
+    AdvEngineID UInt8,
+    PlaceID Int32,
+    RefererCategories Array(UInt16),
+    URLCategories Array(UInt16),
+    URLRegions Array(UInt32),
+    RefererRegions Array(UInt32),
+    IsYandex UInt8,
+    GoalReachesDepth Int32,
+    GoalReachesURL Int32,
+    GoalReachesAny Int32,
+    SocialSourceNetworkID UInt8,
+    SocialSourcePage String,
+    MobilePhoneModel String,
+    ClientEventTime DateTime,
+    RegionID UInt32,
+    ClientIP UInt32,
+    ClientIP6 FixedString(16),
+    RemoteIP UInt32,
+    RemoteIP6 FixedString(16),
+    IPNetworkID UInt32,
+    SilverlightVersion3 UInt32,
+    CodeVersion UInt32,
+    ResolutionWidth UInt16,
+    ResolutionHeight UInt16,
+    UserAgentMajor UInt16,
+    UserAgentMinor UInt16,
+    WindowClientWidth UInt16,
+    WindowClientHeight UInt16,
+    SilverlightVersion2 UInt8,
+    SilverlightVersion4 UInt16,
+    FlashVersion3 UInt16,
+    FlashVersion4 UInt16,
+    ClientTimeZone Int16,
+    OS UInt8,
+    UserAgent UInt8,
+    ResolutionDepth UInt8,
+    FlashMajor UInt8,
+    FlashMinor UInt8,
+    NetMajor UInt8,
+    NetMinor UInt8,
+    MobilePhone UInt8,
+    SilverlightVersion1 UInt8,
+    Age UInt8,
+    Sex UInt8,
+    Income UInt8,
+    JavaEnable UInt8,
+    CookieEnable UInt8,
+    JavascriptEnable UInt8,
+    IsMobile UInt8,
+    BrowserLanguage UInt16,
+    BrowserCountry UInt16,
+    Interests UInt16,
+    Robotness UInt8,
+    GeneralInterests Array(UInt16),
+    Params Array(String),
+    "Goals.ID" Array(UInt32),
+    "Goals.Serial" Array(UInt32),
+    "Goals.EventTime" Array(DateTime),
+    "Goals.Price" Array(Int64),
+    "Goals.OrderID" Array(String),
+    "Goals.CurrencyID" Array(UInt32),
+    WatchIDs Array(UInt64),
+    ParamSumPrice Int64,
+    ParamCurrency FixedString(3),
+    ParamCurrencyID UInt16,
+    ClickLogID UInt64,
+    ClickEventID Int32,
+    ClickGoodEvent Int32,
+    ClickEventTime DateTime,
+    ClickPriorityID Int32,
+    ClickPhraseID Int32,
+    ClickPageID Int32,
+    ClickPlaceID Int32,
+    ClickTypeID Int32,
+    ClickResourceID Int32,
+    ClickCost UInt32,
+    ClickClientIP UInt32,
+    ClickDomainID UInt32,
+    ClickURL String,
+    ClickAttempt UInt8,
+    ClickOrderID UInt32,
+    ClickBannerID UInt32,
+    ClickMarketCategoryID UInt32,
+    ClickMarketPP UInt32,
+    ClickMarketCategoryName String,
+    ClickMarketPPName String,
+    ClickAWAPSCampaignName String,
+    ClickPageName String,
+    ClickTargetType UInt16,
+    ClickTargetPhraseID UInt64,
+    ClickContextType UInt8,
+    ClickSelectType Int8,
+    ClickOptions String,
+    ClickGroupBannerID Int32,
+    OpenstatServiceName String,
+    OpenstatCampaignID String,
+    OpenstatAdID String,
+    OpenstatSourceID String,
+    UTMSource String,
+    UTMMedium String,
+    UTMCampaign String,
+    UTMContent String,
+    UTMTerm String,
+    FromTag String,
+    HasGCLID UInt8,
+    FirstVisit DateTime,
+    PredLastVisit Date,
+    LastVisit Date,
+    TotalVisits UInt32,
+    "TraficSource.ID" Array(Int8),
+    "TraficSource.SearchEngineID" Array(UInt16),
+    "TraficSource.AdvEngineID" Array(UInt8),
+    "TraficSource.PlaceID" Array(UInt16),
+    "TraficSource.SocialSourceNetworkID" Array(UInt8),
+    "TraficSource.Domain" Array(String),
+    "TraficSource.SearchPhrase" Array(String),
+    "TraficSource.SocialSourcePage" Array(String),
+    Attendance FixedString(16),
+    CLID UInt32,
+    YCLID UInt64,
+    NormalizedRefererHash UInt64,
+    SearchPhraseHash UInt64,
+    RefererDomainHash UInt64,
+    NormalizedStartURLHash UInt64,
+    StartURLDomainHash UInt64,
+    NormalizedEndURLHash UInt64,
+    TopLevelDomain UInt64,
+    URLScheme UInt64,
+    OpenstatServiceNameHash UInt64,
+    OpenstatCampaignIDHash UInt64,
+    OpenstatAdIDHash UInt64,
+    OpenstatSourceIDHash UInt64,
+    UTMSourceHash UInt64,
+    UTMMediumHash UInt64,
+    UTMCampaignHash UInt64,
+    UTMContentHash UInt64,
+    UTMTermHash UInt64,
+    FromHash UInt64,
+    WebVisorEnabled UInt8,
+    WebVisorActivity UInt32,
+    "ParsedParams.Key1" Array(String),
+    "ParsedParams.Key2" Array(String),
+    "ParsedParams.Key3" Array(String),
+    "ParsedParams.Key4" Array(String),
+    "ParsedParams.Key5" Array(String),
+    "ParsedParams.ValueDouble" Array(Float64),
+    "Market.Type" Array(UInt8),
+    "Market.GoalID" Array(UInt32),
+    "Market.OrderID" Array(String),
+    "Market.OrderPrice" Array(Int64),
+    "Market.PP" Array(UInt32),
+    "Market.DirectPlaceID" Array(UInt32),
+    "Market.DirectOrderID" Array(UInt32),
+    "Market.DirectBannerID" Array(UInt32),
+    "Market.GoodID" Array(String),
+    "Market.GoodName" Array(String),
+    "Market.GoodQuantity" Array(Int32),
+    "Market.GoodPrice" Array(Int64),
+    IslandID FixedString(16)
+)
+ENGINE = CollapsingMergeTree(Sign)
+PARTITION BY toYYYYMM(StartDate)
+ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
+SAMPLE BY intHash32(UserID)
+SETTINGS disk = disk(type = cache, path = '/var/lib/clickhouse/filesystem_caches/', max_size = '4G',
+         disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/'));

docker/test/stateful/run.sh

@@ -97,21 +97,9 @@ start
 
 setup_logs_replication
 
-# shellcheck disable=SC2086 # No quotes because I want to split it into words.
-/s3downloader --url-prefix "$S3_URL" --dataset-names $DATASETS
-chmod 777 -R /var/lib/clickhouse
 clickhouse-client --query "SHOW DATABASES"
-
-clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
-
-service clickhouse-server restart
-
-# Wait for server to start accepting connections
-for _ in {1..120}; do
-    clickhouse-client --query "SELECT 1" && break
-    sleep 1
-done
-
+clickhouse-client --query "CREATE DATABASE datasets"
+clickhouse-client --multiquery < create.sql
 clickhouse-client --query "SHOW TABLES FROM datasets"
 
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then