Merge eb00c89 into a9eee10

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.5.1] - 2023-01-06
+### Fixed
+- Check for `expired` property before disabling tables from TTL feature rather than the `unreferenced` property.
+
 ## [3.5.0] - 2023-01-04
 ### Fixed
 - Allow cleanup delays to be specified in months or years as well as smaller units by combining the Period and Duration specifications.

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2022 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2022 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java

@@ -57,14 +57,14 @@ public Table createTable(String path, String tableName, boolean partitioned) thr
   }
 
   public Table createTable(String path, String tableName, boolean partitioned, boolean withBeekeeperProperty)
-      throws TException {
+    throws TException {
     Table hiveTable = new Table();
     hiveTable.setDbName(DATABASE_NAME_VALUE);
     hiveTable.setTableName(tableName);
     hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
     hiveTable.putToParameters("EXTERNAL", "TRUE");
     if (withBeekeeperProperty) {
-      hiveTable.putToParameters(LifecycleEventType.UNREFERENCED.getTableParameterName(), "true");
+      hiveTable.putToParameters(LifecycleEventType.EXPIRED.getTableParameterName(), "true");
     }
     if (partitioned) {
       hiveTable.setPartitionKeys(PARTITION_COLUMNS);
@@ -88,24 +88,18 @@ public Table createTable(String path, String tableName, boolean partitioned, boo
    * @param hiveTable Table to add partitions to
    * @param partitionValues The list of partition values, e.g. ["2020-01-01", "0", "A"]
    * @throws Exception May be thrown if there is a problem when trying to write the data to the file, or when the client
-   *                   adds the partition to the table.
+   *           adds the partition to the table.
    */
-  public void addPartitionsToTable(
-      String path,
-      Table hiveTable,
-      List<String> partitionValues)
-      throws Exception {
+  public void addPartitionsToTable(String path, Table hiveTable, List<String> partitionValues) throws Exception {
     String eventDate = "/event_date=" + partitionValues.get(0); // 2020-01-01
     String eventHour = eventDate + "/event_hour=" + partitionValues.get(1); // 0
     String eventType = eventHour + "/event_type=" + partitionValues.get(2); // A
     URI partitionUri = URI.create(path + eventType);
 
     metastoreClient
-        .add_partitions(
-            Collections
-                .singletonList(newTablePartition(hiveTable,
-                    List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)),
-                    partitionUri)));
+        .add_partitions(Collections
+            .singletonList(newTablePartition(hiveTable,
+                List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)), partitionUri)));
   }
 
   private Partition newTablePartition(Table hiveTable, List<String> values, URI location) {

beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/service/MetadataDisableTablesService.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2021 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 package com.expediagroup.beekeeper.metadata.cleanup.service;
 
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED;
-import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,7 +45,8 @@ public class MetadataDisableTablesService implements DisableTablesService {
 
   public MetadataDisableTablesService(
       CleanerClientFactory cleanerClientFactory,
-      HousekeepingMetadataRepository housekeepingMetadataRepository, boolean dryRunEnabled) {
+      HousekeepingMetadataRepository housekeepingMetadataRepository,
+      boolean dryRunEnabled) {
     this.cleanerClientFactory = cleanerClientFactory;
     this.housekeepingMetadataRepository = housekeepingMetadataRepository;
     this.dryRunEnabled = dryRunEnabled;
@@ -63,8 +64,8 @@ private void handleTable(HousekeepingMetadata table) {
     if (!tableHasBeekeeperProperty(table)) {
       log.info("Disabling table {}.{}", table.getDatabaseName(), table.getTableName());
       if (!dryRunEnabled) {
-        housekeepingMetadataRepository.deleteScheduledOrFailedPartitionRecordsForTable(table.getDatabaseName(),
-            table.getTableName());
+        housekeepingMetadataRepository
+            .deleteScheduledOrFailedPartitionRecordsForTable(table.getDatabaseName(), table.getTableName());
         table.setHousekeepingStatus(DISABLED);
         housekeepingMetadataRepository.save(table);
       }
@@ -73,9 +74,8 @@ private void handleTable(HousekeepingMetadata table) {
 
   private boolean tableHasBeekeeperProperty(HousekeepingMetadata metadata) {
     try (CleanerClient client = cleanerClientFactory.newInstance()) {
-      Map<String, String> properties = client.getTableProperties(metadata.getDatabaseName(),
-          metadata.getTableName());
-      String beekeeperProperty = properties.get(UNREFERENCED.getTableParameterName());
+      Map<String, String> properties = client.getTableProperties(metadata.getDatabaseName(), metadata.getTableName());
+      String beekeeperProperty = properties.get(EXPIRED.getTableParameterName());
       return "true".equals(beekeeperProperty);
     } catch (IOException e) {
       throw new BeekeeperException("Can't instantiate cleaner client.", e);

beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/MetadataDisableTablesServiceTest.java

@@ -22,7 +22,6 @@
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED;
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
 import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED;
-import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -72,7 +71,7 @@ public class MetadataDisableTablesServiceTest {
   @BeforeEach
   public void init() {
     Map<String, String> properties = new HashMap<>();
-    properties.put(UNREFERENCED.getTableParameterName(), "true");
+    properties.put(EXPIRED.getTableParameterName(), "true");
     when(hiveClient.getTableProperties(Mockito.any(), Mockito.any())).thenReturn(properties);
     when(hiveClientFactory.newInstance()).thenReturn(hiveClient);
     disableTablesService = new MetadataDisableTablesService(hiveClientFactory, metadataRepository, false);