Merge pull request #68849 from abrennan89/152-411

[enterprise-4.11] OBSDOCS-152: Collate storage docs and remove duplicated files

_topic_maps/_topic_map.yml

@@ -2277,8 +2277,6 @@ Topics:
   Dir: config
   Distros: openshift-enterprise,openshift-origin
   Topics:
-  - Name: Configuring the log store
-    File: cluster-logging-log-store
   - Name: Configuring CPU and memory limits for Logging components
     File: cluster-logging-memory
   - Name: Using tolerations to control Logging pod placement
@@ -2287,8 +2285,6 @@ Topics:
     File: cluster-logging-moving-nodes
   - Name: Configuring systemd-journald for Logging
     File: cluster-logging-systemd
-- Name: Logging using LokiStack
-  File: cluster-logging-loki
 - Name: Log collection and forwarding
   Dir: log_collection_forwarding
   Topics:
@@ -2302,6 +2298,17 @@ Topics:
     File: cluster-logging-collector
   - Name: Collecting and storing Kubernetes events
     File: cluster-logging-eventrouter
+- Name: Log storage
+  Dir: log_storage
+  Topics:
+  - Name: About log storage
+    File: about-log-storage
+  - Name: Installing log storage
+    File: installing-log-storage
+  - Name: Configuring the LokiStack log store
+    File: cluster-logging-loki
+  - Name: Configuring the Elasticsearch log store
+    File: logging-config-es-store
 - Name: Logging alerts
   Dir: logging_alerts
   Topics:

logging/cluster-logging-deploying.adoc

@@ -23,21 +23,15 @@ endif::[]
 
 include::snippets/logging-fluentd-dep-snip.adoc[]
 
+//Installing the Red Hat OpenShift Logging Operator via webconsole
 include::modules/cluster-logging-deploy-console.adoc[leveloffset=+1]
 
 include::modules/cluster-logging-deploy-cli.adoc[leveloffset=+1]
 
-[id="cluster-logging-deploying-es-operator"]
-== Installing the Elasticsearch Operator
-
-include::snippets/logging-elastic-dep-snip.adoc[]
-include::modules/logging-es-storage-considerations.adoc[leveloffset=+2]
-include::modules/logging-install-es-operator.adoc[leveloffset=+2]
-
 [role="_additional-resources"]
 .Additional resources
 * xref:../operators/admin/olm-adding-operators-to-cluster.adoc#olm-installing-operators-from-operatorhub_olm-adding-operators-to-a-cluster[Installing Operators from the OperatorHub]
-* xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-removing-unused-components-if-no-elasticsearch_cluster-logging-log-store[Removing unused components if you do not use the default Elasticsearch log store]
+* xref:../logging/log_storage/logging-config-es-store.adoc#cluster-logging-removing-unused-components-if-no-elasticsearch_logging-config-es-store[Removing unused components if you do not use the default Elasticsearch log store]
 
 [id="cluster-logging-deploying-postinstallation"]
 == Postinstallation tasks

logging/cluster-logging.adoc

@@ -17,7 +17,7 @@ The Operators are responsible for deploying, upgrading, and maintaining the {log
 
 [NOTE]
 ====
-Because the internal {product-title} Elasticsearch log store does not provide secure storage for audit logs, audit logs are not stored in the internal Elasticsearch instance by default. If you want to send the audit logs to the default internal Elasticsearch log store, for example to view the audit logs in Kibana, you must use the Log Forwarding API as described in xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
+Because the internal {product-title} Elasticsearch log store does not provide secure storage for audit logs, audit logs are not stored in the internal Elasticsearch instance by default. If you want to send the audit logs to the default internal Elasticsearch log store, for example to view the audit logs in Kibana, you must use the Log Forwarding API as described in xref:../logging/log_storage/logging-config-es-store.adoc#cluster-logging-elasticsearch-audit_logging-config-es-store[Forward audit logs to the log store].
 ====
 
 include::modules/logging-architecture-overview.adoc[leveloffset=+1]
@@ -46,10 +46,6 @@ include::modules/cluster-logging-export-fields.adoc[leveloffset=+2]
 
 For information, see xref:../logging/cluster-logging-exported-fields.adoc#cluster-logging-exported-fields[About exporting fields].
 
-include::modules/cluster-logging-about-logstore.adoc[leveloffset=+2]
-
-For information, see xref:../logging/config/cluster-logging-log-store.adoc#cluster-logging-log-store[Configuring the log store].
-
 include::modules/cluster-logging-eventrouter-about.adoc[leveloffset=+2]
 
 For information, see xref:../logging/log_collection_forwarding/cluster-logging-eventrouter.adoc#cluster-logging-eventrouter[Collecting and storing Kubernetes events].

logging/log_collection_forwarding/log-forwarding.adoc

@@ -36,7 +36,7 @@ By default, the {logging} sends container and infrastructure logs to the default
 
 [NOTE]
 ====
-To send audit logs to the internal Elasticsearch log store, use the Cluster Log Forwarder as described in xref:../../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Forward audit logs to the log store].
+To send audit logs to the internal Elasticsearch log store, use the Cluster Log Forwarder as described in xref:../../logging/log_storage/logging-config-es-store.adoc#cluster-logging-elasticsearch-audit_logging-config-es-store[Forwarding audit logs to the log store].
 ====
 
 include::modules/cluster-logging-collector-log-forwarding-about.adoc[leveloffset=+1]

logging/log_storage/_attributes

@@ -0,0 +1 @@
+../../_attributes/

logging/log_storage/about-log-storage.adoc

@@ -0,0 +1,30 @@
+:_mod-docs-content-type: ASSEMBLY
+include::_attributes/common-attributes.adoc[]
+include::_attributes/attributes-openshift-dedicated.adoc[]
+[id="about-log-storage"]
+= About log storage
+:context: about-log-storage
+
+toc::[]
+
+You can use an internal Loki or Elasticsearch log store on your cluster for storing logs, or you can use a xref:../../logging/log_collection_forwarding/log-forwarding.adoc#logging-create-clf_log-forwarding[`ClusterLogForwarder` custom resource (CR)] to forward logs to an external store.
+
+[id="log-storage-overview-types"]
+== Log storage types
+
+Loki is a horizontally scalable, highly available, multi-tenant log aggregation system offered as an alternative to Elasticsearch as a log store for the {logging}.
+
+Elasticsearch indexes incoming log records completely during ingestion. Loki only indexes a few fixed labels during ingestion and defers more complex parsing until after the logs have been stored. This means Loki can collect logs more quickly.
+
+include::modules/cluster-logging-about-es-logstore.adoc[leveloffset=+2]
+
+[id="log-storage-overview-querying"]
+== Querying log stores
+
+You can query Loki by using the link:https://grafana.com/docs/loki/latest/logql/[LogQL log query language].
+
+[role="_additional-resources"]
+[id="additional-resources_log-storage-overview"]
+== Additional resources
+* link:https://grafana.com/docs/loki/latest/get-started/components/[Loki components documentation]
+* link:https://loki-operator.dev/docs/object_storage.md/[Loki Object Storage documentation]

logging/cluster-logging-loki.adoc → logging/log_storage/cluster-logging-loki.adoc

@@ -1,33 +1,15 @@
 :_mod-docs-content-type: ASSEMBLY
 :context: cluster-logging-loki
 [id="cluster-logging-loki"]
-= Logging using LokiStack
+= Configuring the LokiStack log store
 include::_attributes/common-attributes.adoc[]
 
 toc::[]
 
 In {logging} documentation, _LokiStack_ refers to the {logging} supported combination of Loki and web proxy with {product-title} authentication integration. LokiStack's proxy uses {product-title} authentication to enforce multi-tenancy. _Loki_ refers to the log store as either the individual component or an external store.
 
-Loki is a horizontally scalable, highly available, multi-tenant log aggregation system currently offered as an alternative to Elasticsearch as a log store for the {logging}. Elasticsearch indexes incoming log records completely during ingestion. Loki only indexes a few fixed labels during ingestion and defers more complex parsing until after the logs have been stored. This means Loki can collect logs more quickly. You can query Loki by using the link:https://grafana.com/docs/loki/latest/logql/[LogQL log query language].
-
-include::modules/loki-deployment-sizing.adoc[leveloffset=+1]
-
-include::modules/cluster-logging-loki-deploy.adoc[leveloffset=+1]
-
-//include::modules/logging-deploy-loki-console.adoc[leveloffset=+1]
-
 include::modules/logging-creating-new-group-cluster-admin-user-role.adoc[leveloffset=+1]
 
-include::modules/logging-loki-gui-install.adoc[leveloffset=+1]
-
-include::modules/logging-clo-gui-install.adoc[leveloffset=+1]
-
-include::modules/logging-loki-cli-install.adoc[leveloffset=+1]
-
-include::modules/logging-clo-cli-install.adoc[leveloffset=+1]
-
-include::modules/configuring-log-storage-cr.adoc[leveloffset=+1]
-
 include::modules/logging-loki-storage.adoc[leveloffset=+1]
 
 include::modules/logging-loki-storage-aws.adoc[leveloffset=+2]

logging/log_storage/images

@@ -0,0 +1 @@
+../../images/

logging/log_storage/installing-log-storage.adoc

@@ -0,0 +1,34 @@
+:_mod-docs-content-type: ASSEMBLY
+include::_attributes/common-attributes.adoc[]
+include::_attributes/attributes-openshift-dedicated.adoc[]
+[id="installing-log-storage"]
+= Installing log storage
+:context: installing-log-storage
+
+toc::[]
+
+You can use the {oc-first} or the {product-title} web console to install a log store on your {product-title} cluster.
+
+include::snippets/logging-elastic-dep-snip.adoc[]
+
+[id="installing-log-storage-loki"]
+== Installing a Loki log store
+
+You can use the {loki-op} to install an internal Loki log store on your {product-title} cluster.
+
+include::modules/loki-deployment-sizing.adoc[leveloffset=+2]
+include::modules/logging-loki-gui-install.adoc[leveloffset=+2]
+include::modules/logging-loki-cli-install.adoc[leveloffset=+2]
+
+[id="installing-log-storage-es"]
+== Installing an Elasticsearch log store
+
+You can use the {es-op} to install an internal Elasticsearch log store on your {product-title} cluster.
+
+include::snippets/logging-elastic-dep-snip.adoc[]
+include::modules/logging-es-storage-considerations.adoc[leveloffset=+2]
+include::modules/logging-install-es-operator.adoc[leveloffset=+2]
+include::modules/cluster-logging-deploy-cli.adoc[leveloffset=+2]
+
+// configuring log store in the clusterlogging CR
+include::modules/configuring-log-storage-cr.adoc[leveloffset=+1]

logging/config/cluster-logging-log-store.adoc → logging/log_storage/logging-config-es-store.adoc

@@ -1,26 +1,27 @@
 :_mod-docs-content-type: ASSEMBLY
-[id="cluster-logging-log-store"]
-= Configuring the log store
+[id="logging-config-es-store"]
+= Configuring the Elasticsearch log store
 include::_attributes/common-attributes.adoc[]
 include::_attributes/attributes-openshift-dedicated.adoc[]
-:context: cluster-logging-log-store
+:context: logging-config-es-store
 
 toc::[]
 
-{logging-title-uc} uses Elasticsearch 6 (ES) to store and organize the log data.
+You can use Elasticsearch 6 to store and organize log data.
 
 You can make modifications to your log store, including:
 
-* storage for your Elasticsearch cluster
-* shard replication across data nodes in the cluster, from full replication to no replication
-* external access to Elasticsearch data
+* Storage for your Elasticsearch cluster
+* Shard replication across data nodes in the cluster, from full replication to no replication
+* External access to Elasticsearch data
+
+include::modules/configuring-log-storage-cr.adoc[leveloffset=+1]
 
 include::modules/cluster-logging-elasticsearch-audit.adoc[leveloffset=+1]
 
 [role="_additional-resources"]
 .Additional resources
-
-* For more information on the Log Forwarding API, see xref:../../logging/log_collection_forwarding/log-forwarding.adoc#log-forwarding[Forwarding logs using the Log Forwarding API].
+* xref:../../logging/log_collection_forwarding/log-forwarding.adoc#log-forwarding[About log collection and forwarding]
 
 include::modules/cluster-logging-elasticsearch-retention.adoc[leveloffset=+1]
 

logging/log_storage/modules

@@ -0,0 +1 @@
+../../modules/

logging/log_storage/snippets

@@ -0,0 +1 @@
+../../snippets/

logging/log_visualization/logging-kibana.adoc

@@ -19,7 +19,7 @@ Use and configuration of the Kibana interface is beyond the scope of this docume
 
 [NOTE]
 ====
-The audit logs are not stored in the internal {product-title} Elasticsearch instance by default. To view the audit logs in Kibana, you must use the xref:../../logging/config/cluster-logging-log-store.adoc#cluster-logging-elasticsearch-audit_cluster-logging-log-store[Log Forwarding API] to configure a pipeline that uses the `default` output for audit logs.
+The audit logs are not stored in the internal {product-title} Elasticsearch instance by default. To view the audit logs in Kibana, you must use the xref:../../logging/log_storage/logging-config-es-store.adoc#cluster-logging-elasticsearch-audit_logging-config-es-store[Log Forwarding API] to configure a pipeline that uses the `default` output for audit logs.
 ====
 
 include::modules/cluster-logging-visualizer-indices.adoc[leveloffset=+1]

modules/cluster-logging-about-logstore.adoc → modules/cluster-logging-about-es-logstore.adoc

@@ -3,10 +3,8 @@
 // * logging/cluster-logging.adoc
 
 :_mod-docs-content-type: CONCEPT
-[id="cluster-logging-about-logstore_{context}"]
-= About the log store
-
-By default, {product-title} uses link:https://www.elastic.co/products/elasticsearch[Elasticsearch (ES)] to store log data. Optionally you can use the Log Forwarder API to forward logs to an external store. Several types of store are supported, including fluentd, rsyslog, kafka and others.
+[id="cluster-logging-about-es-logstore_{context}"]
+= About the Elasticsearch log store
 
 The {logging} Elasticsearch instance is optimized and tested for short term storage, approximately seven days. If you want to retain your logs over a longer term, it is recommended you move the data to a third-party storage system.
 

modules/cluster-logging-manual-rollout-rolling.adoc

@@ -1,6 +1,6 @@
 // Module included in the following assemblies:
 //
-// * logging/config/cluster-logging-log-store.adoc
+// * logging/log_storage/logging-config-es-store.adoc
 
 :_mod-docs-content-type: PROCEDURE
 [id="cluster-logging-manual-rollout-rolling_{context}"]