introduction.Rmd

---
title: "Introduction"
---

```{r, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

```{r prepare, include = FALSE}
lc <- laminr::import_module("lamin_cli")

# Restore the current user at the end of the vignette
current_user <- laminr::get_current_lamin_user()
lc$logout()
withr::defer({
  lc <- laminr::import_module("lamin_cli")
  lc$login(current_user)
})

# Set up a temporary test instance
laminr::use_temporary_instance(name = "laminr-intro", modules = c("bionty"))
```

This vignette introduces the **{laminr}** workflow but reproducing examples from the LaminDB documentation.

To learn more about LaminDB, see [docs.lamin.ai](https://docs.lamin.ai).

# Introduction

## Setup

Install the **{laminr}** package.

```r
install.packages("laminr", dependencies = TRUE)
```

Create a LaminDB instance:

```
lc <- laminr::import_module("lamin_cli")
lc$init(storage = "./lamin-tutorial", modules = c("bionty"))
```

Or if you have write access to an instance, login and connect to it:

```
lc$login()
lc$connect("<account>/<instance>")
```

Alternatively, connect after importing the **lamindb** module:

```
ln <- laminr::import_module("lamindb")
ln$connect("<account>/<instance>")
```

## Quickstart

In an R session, transfer an scRNA-seq dataset from the `laminlabs/cellxgene` instance, compute marker genes with **{Seurat}**, and save results.

```{r file = "../../test-docs/r-quickstart.R", eval = FALSE}
```

If you did not use RStudio’s notebook mode, create an html export and then run the following.

```r
lc$save("my-analyis.Rmd")  # save source code and html report for a `.qmd` or `.Rmd` file
```

# Tutorial

See https://docs.lamin.ai/tutorial

## Track notebooks & scripts

See https://docs.lamin.ai/tutorial#track-notebooks-scripts

```{r track}
library("laminr")
ln <- import_module("lamindb")
ln$track()

ln$Transform$to_dataframe()
ln$Run$to_dataframe()
```

## Manage artifacts

See https://docs.lamin.ai/tutorial#manage-artifacts

### Create an artifact

See https://docs.lamin.ai/tutorial#create-an-artifact

```{r create-artifact}
df <- ln$examples$datasets$mini_immuno$get_dataset1(with_typo = TRUE)
df

artifact <- ln$Artifact$from_dataframe(df, key = "my_datasets/rnaseq1.parquet")$save() # create and save
artifact$describe() # describe
```

### Access artifacts

See https://docs.lamin.ai/tutorial#access-artifacts

```{r access-artifacts}
artifact <- ln$Artifact$get(key = "my_datasets/rnaseq1.parquet")
artifact$load()
artifact$cache()
```

### Trace data lineage

See https://docs.lamin.ai/tutorial#trace-data-lineage

```{r trace-lineage}
artifact$transform
artifact$run
artifact$view_lineage()

# ln$finish() # nolint
```

### Annotate an artifact

See https://docs.lamin.ai/tutorial#annotate-an-artifact

```{r annotate-artifact}
# create a label
my_experiment <- ln$Record(name = "My experiment")$save()

# annotate the artifact with a label
artifact$records$add(my_experiment)

# describe the artifact
artifact$describe()

ln$Artifact$filter(records = my_experiment)$to_dataframe()

bt <- import_module("bionty")

# create a cell type label from the source ontology
cell_type <- bt$CellType$from_source(name = "effector T cell")$save()

# annotate the artifact with a cell type
artifact$cell_types$add(cell_type)

# describe the artifact
artifact$describe()

ln$Artifact$filter(cell_types = cell_type)$to_dataframe()

# define the "temperature" & "experiment" features
ln$Feature(name = "temperature", dtype = "float")$save()
ln$Feature(name = "experiment", dtype = ln$Record)$save()

# annotate the artifact
artifact$features$add_values(
  list(temperature = 21.6, experiment = "My experiment")
)

# describe the artifact
artifact$describe()

ln$Artifact$filter(temperature = 21.6)$to_dataframe()
```

### Validate an artifact

See https://docs.lamin.ai/tutorial#validate-an-artifact

```{r validate-artifact}
bt <- import_module("bionty") # <-- use bionty to access registries with imported public ontologies

# define a few more valid labels
ln$Record(name = "DMSO")$save()
ln$Record(name = "IFNG")$save()

# define a few more valid features
ln$Feature(name = "perturbation", dtype = ln$Record)$save()
ln$Feature(name = "cell_type_by_model", dtype = bt$CellType)$save()
ln$Feature(name = "cell_type_by_expert", dtype = bt$CellType)$save()
ln$Feature(name = "assay_oid", dtype = bt$ExperimentalFactor$ontology_id)$save()
ln$Feature(name = "donor", dtype = "str", nullable = TRUE)$save()
ln$Feature(name = "concentration", dtype = "str")$save()
ln$Feature(name = "treatment_time_h", dtype = "num", coerce_dtype = TRUE)$save()

# define a schema that merely enforces a feature identifier type
schema <- ln$Schema(itype = ln$Feature)$save()

testthat::expect_error(
  artifact <- ln$Artifact$from_dataframe(df, key = "my_datasets/rnaseq1.parquet", schema = schema)
)
```

### Make a new version of an artifact

See https://docs.lamin.ai/tutorial#make-a-new-version-of-an-artifact

```{r version-artifact}
# fix the "IFNJ" typo
levels(df$perturbation) <- c("DMSO", "IFNG")
df["sample2", "perturbation"] <- "IFNG"

# create a new version
artifact <- ln$Artifact$from_dataframe(df, key = "my_datasets/rnaseq1.parquet", schema = schema)$save()

# see the annotations
artifact$describe()

# simplest way to check that artifact was validated
artifact$schema

# see all versions of an artifact
artifact$versions$to_dataframe()
```

## Query & search registries

See https://docs.lamin.ai/tutorial#query-search-registries

```{r query-registries}
ln$Artifact$to_dataframe()
ln$Artifact$to_dataframe(include = "features")
ln$Artifact
ln$view()

# get a single record (here the current notebook)
transform <- ln$Transform$get(key = "introduction.Rmd")

# get a set of records by filtering for a directory (LaminDB treats directories
# like AWS S3, as the prefix of the storage key)
ln$Artifact$filter(key__startswith = "my_datasets/")$to_dataframe()

# query all artifacts ingested from a transform
artifacts <- ln$Artifact$filter(transform = transform)$all()

# query all artifacts ingested from a notebook with "tutor" in the description
artifacts <- ln$Artifact$filter(
  transform__description__icontains = "tutor",
)$all()

# search artifacts
ln$Artifact$search("iris")$to_dataframe()

# search transforms
ln$Transform$search("tutor")$to_dataframe()

# look up records with auto-complete
ulabels <- ln$Record$lookup()
```

## Manage files & folders

See https://docs.lamin.ai/tutorial#manage-files-folders

```{r manage-files}
# we use anon=True here in case no aws credentials are configured
ln$UPath("s3://lamindata/iris_studies", anon = TRUE)$view_tree()

artifact <- ln$Artifact("s3://lamindata/iris_studies/study0_raw_images")$save()
artifact

artifact$path
ln$Storage$to_dataframe()
```

## Manage biological registries

See https://docs.lamin.ai/tutorial#manage-biological-registries

```{r manage-registries}
bt <- import_module("bionty")

cell_type_ontology <- bt$CellType$public()
cell_type_ontology

cell_type_ontology$search("gamma-delta T cell") |> head(n = 2)

# create an ontology-coupled cell type record and save it
neuron <- bt$CellType$from_source(name = "neuron")$save()

# create a record to track a new cell state
new_cell_state <- bt$CellType(
  name = "my neuron cell state", description = "explains X"
)$save()

# express that it's a neuron state
new_cell_state$parents$add(neuron)

# view ontological hierarchy
new_cell_state$view_parents(distance = 2)
```

## Manage AnnData object

See https://docs.lamin.ai/tutorial#manage-anndata-objects

```{r manage-anndata}
# define var schema
var_schema <- ln$Schema(itype = bt$Gene$ensembl_gene_id, dtype = "int")$save()

# define composite schema
anndata_schema <- ln$Schema(
  otype = "AnnData", slots = list("obs" = schema, "var.T" = var_schema)
)$save()

ad <- import_module("anndata")

# store the dataset as an AnnData object to distinguish data from metadata
adata <- ad$AnnData(df[, 1:3], obs = df[, 4:(ncol(df) - 1)])

# save curated artifact
artifact <- ln$Artifact$from_anndata(
  adata,
  key = "my_datasets/my_rnaseq1.h5ad", schema = anndata_schema
)$save()
artifact$describe()

# query for all schemas that contain CD8A
schemas <- ln$Schema$filter(genes__symbol = "CD8A")$all()

# query for all artifacts linked to these feature sets
ln$Artifact$filter(schemas__in = schemas)$to_dataframe()
```

## Scale learning

See https://docs.lamin.ai/tutorial#scale-learning

```{r scale-learning}
# a new dataset
df2 <- ln$examples$datasets$mini_immuno$get_dataset2(otype = "DataFrame")
adata <- ad$AnnData(df2[, 1:3], obs = df2[, 4:(ncol(df2) - 1)])
artifact2 <- ln$Artifact$from_anndata(
  adata,
  key = "my_datasets/my_rnaseq2.h5ad", schema = anndata_schema
)$save()

collection <- ln$Collection(list(artifact, artifact2), key = "my-RNA-seq-collection")$save()
collection$describe()
collection$view_lineage()

# if it's small enough, you can load the entire collection into memory as if it was one
collection$load()

# typically, it's too big, hence, open it for streaming (if the backend allows it)
# collection.open() # nolint

# or iterate over its artifacts
collection$artifacts$all()

# or look at a DataFrame listing the artifacts
collection$artifacts$to_dataframe()
```

# Finish

```{r finish}
ln$finish()
```