TtlIntegrationTest.java

/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.client.fs;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import alluxio.AlluxioTestDirectory;
import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.client.file.FileOutStream;
import alluxio.client.file.FileSystem;
import alluxio.conf.PropertyKey;
import alluxio.grpc.CreateFilePOptions;
import alluxio.grpc.FileSystemMasterCommonPOptions;
import alluxio.grpc.LoadMetadataPType;
import alluxio.grpc.TtlAction;
import alluxio.grpc.WritePType;
import alluxio.heartbeat.HeartbeatContext;
import alluxio.heartbeat.HeartbeatScheduler;
import alluxio.heartbeat.ManuallyScheduleHeartbeat;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.testutils.LocalAlluxioClusterResource;
import alluxio.util.CommonUtils;

import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.File;
import java.util.Arrays;

/**
 * Integration tests for handling file TTLs (times to live).
 */
public class TtlIntegrationTest extends BaseIntegrationTest {
  private static final int TTL_INTERVAL_MS = 50;

  private FileSystem mFileSystem;

  private FileOutStream mOutStream = null;

  protected byte[] mBuffer;

  @ClassRule
  public static ManuallyScheduleHeartbeat sManuallySchedule =
      new ManuallyScheduleHeartbeat(HeartbeatContext.MASTER_TTL_CHECK);

  @Rule
  public TemporaryFolder mFolder = new TemporaryFolder();

  public File mUfs = AlluxioTestDirectory.createTemporaryDirectory("RootUfs");
  private String mLocalUfsPath = mUfs.getAbsolutePath();

  @Rule
  public LocalAlluxioClusterResource mLocalAlluxioClusterResource =
      new LocalAlluxioClusterResource.Builder()
              .setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, mLocalUfsPath)
              .setProperty(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS, TTL_INTERVAL_MS)
              .setProperty(PropertyKey.USER_FILE_METADATA_LOAD_TYPE, LoadMetadataPType.NEVER)
              .build();

  @Before
  public void before() throws Exception {
    mFileSystem = FileSystem.Factory.create();
    mBuffer = new byte[10];
    Arrays.fill(mBuffer, (byte) 'A');
  }

  /**
   * Tests that when many TTLs expire at the same time, files are freed properly.
   */
  @Test
  public void expireManyAfterFree() throws Exception {
    int numFiles = 100;
    AlluxioURI[] files = new AlluxioURI[numFiles];
    for (int i = 0; i < numFiles; i++) {
      files[i] = new AlluxioURI("/fileFree" + i);
      // Only the even-index files should expire.
      long ttl = i % 2 == 0 ? TTL_INTERVAL_MS / 2 : TTL_INTERVAL_MS * 1000;
      mOutStream = mFileSystem.createFile(files[i],
          CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH)
              .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(ttl))
                  .build());
      mOutStream.write(mBuffer, 0, 10);
      mOutStream.close();

      // Delete some of the even files to make sure this doesn't trip up the TTL checker.
      if (i % 20 == 0) {
        mFileSystem.delete(files[i]);
      }
    }
    CommonUtils.sleepMs(2 * TTL_INTERVAL_MS);
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK);
    for (int i = 0; i < numFiles; i++) {
      if (i % 2 == 0) {
        if (i % 20 != 0) {
          assertEquals(Constants.NO_TTL, mFileSystem.getStatus(files[i]).getTtl());
          assertEquals(TtlAction.DELETE, mFileSystem.getStatus(files[i]).getTtlAction());
          assertEquals(0, mFileSystem.getStatus(files[i]).getInMemoryPercentage());
          String fileName = "fileFree" + i;
          // Check Ufs file existence
          assertTrue(Arrays.stream(mUfs.list()).anyMatch(s -> s.equals(fileName)));
        }
      } else {
        assertTrue(mFileSystem.exists(files[i]));
        assertEquals(100, mFileSystem.getStatus(files[i]).getInMemoryPercentage());
      }
    }
  }

  /**
   * Tests that when many TTLs expire at the same time, files are deleted from Alluxio and
   * UFS properly.
   */
  @Test
  public void expireManyAfterDelete() throws Exception {
    int numFiles = 100;
    AlluxioURI[] files = new AlluxioURI[numFiles];
    for (int i = 0; i < numFiles; i++) {
      files[i] = new AlluxioURI("/fileDelete" + i);
      // Only the even-index files should expire.
      long ttl = i % 2 == 0 ? TTL_INTERVAL_MS / 2 : TTL_INTERVAL_MS * 1000;
      mOutStream = mFileSystem.createFile(files[i],
              CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH)
                      .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(ttl)
                              .setTtlAction(TtlAction.DELETE)).build());
      mOutStream.write(mBuffer, 0, 10);
      mOutStream.close();

      // Delete some of the even files to make sure this doesn't trip up the TTL checker.
      if (i % 20 == 0) {
        mFileSystem.delete(files[i]);
      }
    }
    CommonUtils.sleepMs(4 * TTL_INTERVAL_MS);
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK);
    for (int i = 0; i < numFiles; i++) {
      if (i % 2 == 0) {
        assertFalse(mFileSystem.exists(files[i]));
        String fileName = "fileDelete" + i;
        // Check Ufs file existence
        assertFalse(Arrays.stream(mUfs.list()).anyMatch(s -> s.equals(fileName)));
      } else {
        assertTrue(mFileSystem.exists(files[i]));
      }
    }
  }
}