Add config for 'logging' entity datasource that by default uses the '…

…default' ElasticFacade cluster, also add 'logging' as a valid value for the entity.@use attribute; add example extend-entity to move to logging group (commented) and some comments about migration options; initial implementation of ElasticDatasourceFactory, work still need on find, eli, and value classes

framework/entity/ServerEntities.xml

@@ -18,15 +18,14 @@ along with this software (see the LICENSE.md file). If not, see
     <!-- moqui.server -->
     <!-- ========================================================= -->
 
-    <entity entity-name="ArtifactHit" package="moqui.server" use="nontransactional"
-            cache="never" sequence-bank-size="100">
+    <entity entity-name="ArtifactHit" package="moqui.server" use="logging" cache="never" sequence-bank-size="100">
         <field name="hitId" type="id" is-pk="true"/>
         <field name="visitId" type="id"/>
         <field name="userId" type="id"/>
         <field name="artifactType" type="text-medium"/>
         <field name="artifactSubType" type="text-medium"/>
-        <field name="artifactName" type="text-medium"><description>The name of the artifact hit. For XML Screens
-            request it is "${webapp-name}.${screen-path}"</description></field>
+        <field name="artifactName" type="text-medium">
+            <description>The name of the artifact hit. For XML Screen request it is "${webapp-name}.${screen-path}"</description></field>
         <field name="parameterString" type="text-long"/>
         <field name="startDateTime" type="date-time"/>
         <field name="runningTimeMillis" type="number-float"/>
@@ -43,8 +42,17 @@ along with this software (see the LICENSE.md file). If not, see
         <index name="ARTIFACT_HIT_VST"><index-field name="visitId"/></index>
         <index name="ARTIFACT_HIT_USR"><index-field name="userId"/></index>
     </entity>
-    <entity entity-name="ArtifactHitBin" package="moqui.server" use="nontransactional"
-            cache="never" sequence-bank-size="100">
+    <!-- to use Elastic/OpenSearch for ArtifactHit: add to logging group which is configured by default to use the 'default' cluster via ElasticFacade -->
+    <!-- TODO: make sure this is commented this before merging!! (not great as default config) -->
+    <!-- <extend-entity entity-name="ArtifactHit" package="moqui.server" group="logging"/> -->
+    <!--
+        TODO for migration:
+        - somehow have ArtifactHitDb always in 'transactional' group to read from and write to ArtifactHit in the logging group
+        - some way to clone entity to new name, with different group/etc?
+        - or some way to do entity operations and override the group name at runtime? might be handy for other things too...
+    -->
+
+    <entity entity-name="ArtifactHitBin" package="moqui.server" use="nontransactional" cache="never" sequence-bank-size="100">
         <field name="hitBinId" type="id" is-pk="true"/>
         <field name="artifactType" type="text-medium"/>
         <field name="artifactSubType" type="text-medium"/>

framework/src/main/groovy/org/moqui/impl/entity/EntityDefinition.groovy

@@ -512,6 +512,7 @@ class EntityDefinition {
 
     /** Returns the table name, ie table-name or converted entity-name */
     String getTableName() { return entityInfo.tableName }
+    String getTableNameLowerCase() { return entityInfo.tableNameLowerCase }
     String getFullTableName() { return entityInfo.fullTableName }
     String getSchemaName() { return entityInfo.schemaName }
 

framework/src/main/groovy/org/moqui/impl/entity/EntityJavaUtil.java

@@ -254,7 +254,7 @@ public static class EntityInfo {
         private final EntityDefinition ed;
         private final EntityFacadeImpl efi;
         public final String internalEntityName, fullEntityName, shortAlias, groupName;
-        public final String tableName, schemaName, fullTableName;
+        public final String tableName, tableNameLowerCase, schemaName, fullTableName;
 
         public final EntityDatasourceFactory datasourceFactory;
         public final boolean isEntityDatasourceFactoryImpl;
@@ -308,6 +308,7 @@ public static class EntityInfo {
             String tableNameAttr = internalEntityNode.attribute("table-name");
             if (tableNameAttr == null || tableNameAttr.isEmpty()) tableNameAttr = EntityJavaUtil.camelCaseToUnderscored(internalEntityName);
             tableName = tableNameAttr;
+            tableNameLowerCase = tableName.toLowerCase();
             String schemaNameAttr = datasourceNode != null ? datasourceNode.attribute("schema-name") : null;
             if (schemaNameAttr != null && schemaNameAttr.length() == 0) schemaNameAttr = null;
             schemaName = schemaNameAttr;

framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticDatasourceFactory.groovy

@@ -14,9 +14,11 @@
 package org.moqui.impl.entity.elastic
 
 import groovy.transform.CompileStatic
+import org.moqui.context.ElasticFacade
 import org.moqui.entity.*
 import org.moqui.impl.entity.EntityDefinition
 import org.moqui.impl.entity.EntityFacadeImpl
+import org.moqui.impl.entity.FieldInfo
 import org.moqui.util.MNode
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -42,9 +44,9 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory {
 
     protected EntityFacadeImpl efi
     protected MNode datasourceNode
-    protected String indexPrefix
+    protected String indexPrefix, clusterName
 
-    protected Set<String> checkedEntitySet = new HashSet<String>()
+    protected Set<String> checkedEntityIndexSet = new HashSet<String>()
 
     ElasticDatasourceFactory() { }
 
@@ -57,7 +59,8 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory {
         // init the DataSource
         MNode inlineOtherNode = datasourceNode.first("inline-other")
         inlineOtherNode.setSystemExpandAttributes(true)
-        indexPrefix = inlineOtherNode.attribute("index-prefix")
+        indexPrefix = inlineOtherNode.attribute("index-prefix") ?: ""
+        clusterName = inlineOtherNode.attribute("cluster-name") ?: "default"
 
         return this
     }
@@ -69,30 +72,27 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory {
     @Override
     boolean checkTableExists(String entityName) {
         EntityDefinition ed
-        // just ignore EntityException on getEntityDefinition
         try { ed = efi.getEntityDefinition(entityName) }
         catch (EntityException e) { return false }
-        // may happen if all entity names includes a DB view entity or other that doesn't really exist
         if (ed == null) return false
 
-        String fullEntityName = ed.getFullEntityName()
-        if (checkedEntitySet.contains(fullEntityName)) return true
+        String indexName = getIndexName(ed)
+        if (checkedEntityIndexSet.contains(indexName)) return true
 
-        // TODO
-        // if (TODO) return false
+        ElasticFacade.ElasticClient elasticClient = efi.ecfi.elasticFacade.getClient(clusterName)
+        if (!elasticClient.indexExists(indexName)) return false
 
-        checkedEntitySet.add(fullEntityName)
+        checkedEntityIndexSet.add(indexName)
         return true
     }
     @Override
     boolean checkAndAddTable(String entityName) {
         EntityDefinition ed
-        // just ignore EntityException on getEntityDefinition
-        try { ed = efi.getEntityDefinition(entityName) } catch (EntityException e) { return false }
-        // may happen if all entity names includes a DB view entity or other that doesn't really exist
+        try { ed = efi.getEntityDefinition(entityName) }
+        catch (EntityException e) { return false }
         if (ed == null) return false
-
-        // TODO
+        checkCreateDocumentIndex(ed)
+        return true
     }
     @Override
     int checkAndAddAllTables() {
@@ -111,22 +111,63 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory {
 
     @Override
     EntityValue makeEntityValue(String entityName) {
-        EntityDefinition entityDefinition = efi.getEntityDefinition(entityName)
-        if (!entityDefinition) {
-            throw new EntityException("Entity not found for name [${entityName}]")
-        }
-
-        // TODO
-        return null
+        EntityDefinition ed = efi.getEntityDefinition(entityName)
+        if (ed == null) throw new EntityException("Entity not found for name ${entityName}")
+        return new ElasticEntityValue(ed, efi, this)
     }
 
     @Override
     EntityFind makeEntityFind(String entityName) { return new ElasticEntityFind(efi, entityName, this) }
 
     @Override
-    DataSource getDataSource() { return null }
+    DataSource getDataSource() {
+        throw new UnsupportedOperationException("DataSource not supported for ElasticFacade based Entity")
+    }
 
     void checkCreateDocumentIndex(EntityDefinition ed) {
-        // TODO
+        String indexName = getIndexName(ed)
+        if (checkedEntityIndexSet.contains(indexName)) return
+
+        ElasticFacade.ElasticClient elasticClient = efi.ecfi.elasticFacade.getClient(clusterName)
+        if (!elasticClient.indexExists(indexName))
+            elasticClient.createIndex(indexName, makeElasticEntityMapping(ed), null)
+
+        checkedEntityIndexSet.add(indexName)
+    }
+
+    String getIndexName(EntityDefinition ed) {
+        return indexPrefix + ed.getTableNameLowerCase()
+    }
+
+    static final Map<String, String> esEntityTypeMap = [id:'keyword', 'id-long':'keyword', date:'date', time:'keyword',
+            'date-time':'date', 'number-integer':'long', 'number-decimal':'double', 'number-float':'double',
+            'currency-amount':'double', 'currency-precise':'double', 'text-indicator':'keyword', 'text-short':'text',
+            'text-medium':'text', 'text-intermediate':'text', 'text-long':'text', 'text-very-long':'text', 'binary-very-long':'binary']
+    static final Set<String> esEntityIsKeywordSet = esEntityTypeMap.findAll({"keyword".equals(it.value)}).keySet()
+    static final Set<String> esEntityAddKeywordSet = new HashSet<>(['text-short', 'text-medium', 'text-intermediate'])
+
+    static Map makeElasticEntityMapping(EntityDefinition ed) {
+        Map<String, Object> rootProperties = [_entity:[type:'keyword']] as Map<String, Object>
+        Map<String, Object> mappingMap = [properties:rootProperties] as Map<String, Object>
+
+        FieldInfo[] allFieldInfo = ed.entityInfo.allFieldInfoArray
+        for (int i = 0; i < allFieldInfo.length; i++) {
+            FieldInfo fieldInfo = allFieldInfo[i]
+            rootProperties.put(fieldInfo.name, makeEntityFieldPropertyMap(fieldInfo))
+        }
+
+        return mappingMap
+    }
+    static Map makeEntityFieldPropertyMap(FieldInfo fieldInfo) {
+        String mappingType = esEntityTypeMap.get(fieldInfo.type) ?: 'text'
+        Map<String, Object> propertyMap = new LinkedHashMap<>()
+        propertyMap.put("type", mappingType)
+        if (esEntityAddKeywordSet.contains(fieldInfo.type) && "text".equals(mappingType))
+            propertyMap.put("fields", [keyword: [type: "keyword"]])
+        if ("date-time".equals(fieldInfo.type))
+            propertyMap.format = "date_time||epoch_millis||date_time_no_millis||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss.S||yyyy-MM-dd"
+        else if ("date".equals(fieldInfo.type))
+            propertyMap.format = "date||strict_date_optional_time||epoch_millis"
+        return propertyMap
     }
 }

framework/src/main/resources/MoquiDefaultConf.xml

@@ -488,6 +488,12 @@
             </inline-jdbc>
         </datasource>
 
+        <!-- The logging group and datasource uses the elastic-facade.cluster configuration to talk to Elastic/OpenSearch -->
+        <datasource group-name="logging" runtime-add-missing="true" startup-add-missing="false"
+                object-factory="org.moqui.impl.entity.elastic.ElasticDatasourceFactory">
+            <inline-other index-prefix="logging-" cluster-name="default"/>
+        </datasource>
+
         <!-- The nontransactional group is in the transactional db by default. To use OrientDB use the datasource below.
             NOTE: startup-add-missing=true is required because OrientDB doesn't allow class create in a TX
             NOTE: adding more entities to the nontransactional group is a work in progress, this will not currently run,
@@ -500,10 +506,6 @@
                 object-factory="org.moqui.impl.entity.orientdb.OrientDatasourceFactory">
             <inline-other uri="plocal:${ORIENTDB_HOME}/databases/MoquiNoSql" username="admin" password="admin"/>
         </datasource>
-        <datasource group-name="logging" startup-add-missing="true"
-                object-factory="org.moqui.impl.entity.elastic.ElasticDatasourceFactory">
-            <inline-other index-prefix="logging"/>
-        </datasource>
         -->
 
         <!-- Refer to these explicitly instead of by directory name convention as is done in components.

framework/xsd/entity-definition-3.xsd

@@ -105,6 +105,8 @@ along with this software (see the LICENSE.md file). If not, see
                         configuration data; eventual consistency is okay; typically cached</xs:documentation></xs:annotation></xs:enumeration>
                     <xs:enumeration value="analytical"><xs:annotation><xs:documentation>analytical entities with data
                         typically derived from transactional data</xs:documentation></xs:annotation></xs:enumeration>
+                    <xs:enumeration value="logging"><xs:annotation><xs:documentation>entities used for logging with
+                        non-transactional data that is generated in high volumes and used mostly for auditing and analytics</xs:documentation></xs:annotation></xs:enumeration>
                 </xs:restriction></xs:simpleType>
             </xs:attribute>
             <xs:attribute name="sequence-primary-use-uuid" type="boolean" default="false">