From 0cc54620e29820615cc2d34d881fff6bd7a11f5b Mon Sep 17 00:00:00 2001 From: yuriy-glotanov Date: Tue, 17 Dec 2019 23:03:42 +0300 Subject: [PATCH] performance optimize: persist(), indexes --- src/main/java/su/interference/core/Chunk.java | 1 + .../java/su/interference/core/ChunkMap.java | 27 +++- .../java/su/interference/core/DataChunk.java | 26 ++-- .../java/su/interference/core/DataObject.java | 6 +- .../su/interference/core/IndexElement.java | 36 +---- .../java/su/interference/core/IndexFrame.java | 9 +- .../java/su/interference/core/IndexList.java | 6 +- .../java/su/interference/core/Instance.java | 3 + .../java/su/interference/core/ValueSet.java | 31 ++--- .../su/interference/persistent/Table.java | 124 ++++++++++++++---- .../interference/sql/SQLJoinThreadPool.java | 2 +- 11 files changed, 171 insertions(+), 100 deletions(-) diff --git a/src/main/java/su/interference/core/Chunk.java b/src/main/java/su/interference/core/Chunk.java index 7530ce6..0dfb1f4 100644 --- a/src/main/java/su/interference/core/Chunk.java +++ b/src/main/java/su/interference/core/Chunk.java @@ -43,6 +43,7 @@ public interface Chunk extends Comparable { byte[] getChunk(); void setChunk(byte[] chunk); int getBytesAmount(); + ValueSet getDcs(); Object getEntity() throws ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException; Object getUndoEntity() throws ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException; void updateEntity(Object o) throws InternalException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, MalformedURLException; diff --git a/src/main/java/su/interference/core/ChunkMap.java b/src/main/java/su/interference/core/ChunkMap.java index 2112e14..a03a0c7 100644 --- a/src/main/java/su/interference/core/ChunkMap.java +++ b/src/main/java/su/interference/core/ChunkMap.java @@ -35,20 +35,26 @@ this software and associated documentation files (the "Software"), to deal in public class ChunkMap { private final ConcurrentHashMap hmap; + private final ConcurrentHashMap imap; private final List list; + private volatile boolean sorted; public ChunkMap() { - hmap = new ConcurrentHashMap(); - list = new CopyOnWriteArrayList(); + hmap = new ConcurrentHashMap<>(); + imap = new ConcurrentHashMap<>(); + list = new CopyOnWriteArrayList<>(); } public void sort() { Collections.sort(list); + sorted = true; } public void add(Chunk c) { hmap.put(c.getHeader().getPtr(), c); + imap.put(c.getDcs(), c); list.add(c); + sorted = false; } public List getChunks() { @@ -63,10 +69,16 @@ public Chunk get(int i) { return list.get(i); } + public Chunk getByKey(ValueSet key) { + return imap.get(key); + } + public void removeByPtr(int i) { final boolean x = list.remove(hmap.get(i)); - final Object o = hmap.remove(i); - if (!x || o == null) { + final Chunk c = (Chunk)hmap.remove(i); + imap.remove(c.getDcs()); + sorted = false; + if (!x || c == null) { throw new RuntimeException("Internal error during remove object from frame"); } } @@ -75,6 +87,8 @@ public void remove(int i) { final Chunk c = list.get(i); list.remove(i); hmap.remove(c.getHeader().getPtr()); + imap.remove(c.getDcs()); + sorted = false; } public int size() { @@ -84,6 +98,11 @@ public int size() { public void clear() { hmap.clear(); list.clear(); + imap.clear(); + sorted = false; } + public boolean isSorted() { + return sorted; + } } diff --git a/src/main/java/su/interference/core/DataChunk.java b/src/main/java/su/interference/core/DataChunk.java index 14620eb..0a22d58 100644 --- a/src/main/java/su/interference/core/DataChunk.java +++ b/src/main/java/su/interference/core/DataChunk.java @@ -65,21 +65,25 @@ public class DataChunk implements Chunk { private final CustomSerializer sr = new CustomSerializer(); //returns datacolumn set - public ValueSet getDcs() throws ClassNotFoundException, IllegalAccessException, InternalException, MalformedURLException { + public ValueSet getDcs() { if (dcs==null) { - final Field[] f = t.getFields(); - final Object[] vs = new Object[f.length]; - for (int i=0; i vs = new ArrayList<>(); + for (int i = 0; i < f.length; i++) { + final int m = f[i].getModifiers(); + final Transient ta = f[i].getAnnotation(Transient.class); + if (ta == null) { + if (Modifier.isPrivate(m)) { + f[i].setAccessible(true); + } + vs.add(f[i].get(entity)); } - vs[i] = f[i].get(entity); } + dcs = new ValueSet(vs.toArray(new Object[]{})); + } catch (Exception e) { + throw new RuntimeException(e); } - dcs = new ValueSet(vs); } return dcs; } diff --git a/src/main/java/su/interference/core/DataObject.java b/src/main/java/su/interference/core/DataObject.java index 2ba190e..9789e21 100644 --- a/src/main/java/su/interference/core/DataObject.java +++ b/src/main/java/su/interference/core/DataObject.java @@ -60,14 +60,18 @@ public interface DataObject { long getIdValue(Session s, LLT llt) throws Exception; long getIncValue(Session s, LLT llt) throws Exception; void incFrameAmount (); + Class getSc(); + Class getGenericClass(); boolean isNoTran() throws ClassNotFoundException, MalformedURLException; boolean isIndex() throws ClassNotFoundException, MalformedURLException; Object newInstance() throws IOException, InternalException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException; Object getInstance() throws IOException, InternalException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException; void usedSpace (FrameData bd, int used, boolean persist, Session s, LLT llt); void addIndexValue (DataChunk dc) throws IllegalAccessException, InvocationTargetException, NoSuchMethodException; - Field[] getColumns(); java.lang.reflect.Field[] getFields() throws ClassNotFoundException, InternalException, MalformedURLException; + java.lang.reflect.Field getIdField(); + String getIdFieldType(); + String getIdFieldGetter(); FrameData allocateFrame(DataFile df, DataObject t, Session s, LLT llt) throws Exception; } diff --git a/src/main/java/su/interference/core/IndexElement.java b/src/main/java/su/interference/core/IndexElement.java index 3dbaa3b..4992cae 100644 --- a/src/main/java/su/interference/core/IndexElement.java +++ b/src/main/java/su/interference/core/IndexElement.java @@ -32,19 +32,20 @@ this software and associated documentation files (the "Software"), to deal in */ public class IndexElement implements Comparable { - private IndexElementKey key; - private Object element; + final private IndexElementKey key; + private Object element; + final private boolean ex; - public IndexElement (IndexElementKey key, Object element) { + public IndexElement (IndexElementKey key, Object element, boolean ex) { this.key = key; this.element = element; + this.ex = ex; } public int compareTo(final Object obj) { final IndexElement j = (IndexElement)obj; final int res = this.getKey().compareTo(j.getKey()); - final String clname = this.getElement().getClass().getSimpleName(); - if (clname.equals("Integer")&&res==0) { + if (this.ex && res == 0) { if ((Integer)this.getElement() < (Integer)j.getElement()) { return -1; } else if ((Integer)this.getElement() > (Integer)j.getElement()) { return 1; } return 0; } @@ -55,10 +56,6 @@ public IndexElementKey getKey() { return key; } - public void setKey(IndexElementKey key) { - this.key = key; - } - public Object getElement() { return element; } @@ -68,26 +65,7 @@ public void setElement(Object element) { } public String ElementToString () { - String clname = this.element.getClass().getSimpleName(); - if (clname.equals("Integer")) { - return ""+ (Integer)this.element; - } - if (clname.equals("Long")) { - return ""+ (Long)this.element; - } - if (clname.equals("Float")) { - return ""+ (Float)this.element; - } - if (clname.equals("Double")) { - return ""+ (Double)this.element; - } - if (clname.equals("String")) { - return ""+ (String)this.element; - } - if (clname.equals("Date")) { - return ""+ (Date)this.element; - } - return "Datatype not recognized"; + return "" + this.element; } } diff --git a/src/main/java/su/interference/core/IndexFrame.java b/src/main/java/su/interference/core/IndexFrame.java index 0727f4e..8c946df 100644 --- a/src/main/java/su/interference/core/IndexFrame.java +++ b/src/main/java/su/interference/core/IndexFrame.java @@ -208,7 +208,9 @@ private boolean isFill(DataChunk ie) { } public ValueSet sort() throws ClassNotFoundException, IllegalAccessException, InternalException, MalformedURLException { - this.data.sort(); + if (!this.data.isSorted()) { + this.data.sort(); + } this.sorted = true; if (this.data.size()>0) { return ((DataChunk)this.data.get(this.data.size()-1)).getDcs(); @@ -255,13 +257,16 @@ public synchronized ArrayList getChildElementsPtr(ValueSet value) throws C } //return first element which found - for unique indexes - public DataChunk getObjectByKey(ValueSet key) throws ClassNotFoundException, IllegalAccessException, InternalException, MalformedURLException { + public DataChunk getObjectByKey(ValueSet key) { +/* for (Chunk ie : this.data.getChunks()) { if (((DataChunk)ie).getDcs().equals(key)) { return (DataChunk)ie; } } return null; +*/ + return (DataChunk) this.data.getByKey(key); } //return all element which found - for non-unique indexes diff --git a/src/main/java/su/interference/core/IndexList.java b/src/main/java/su/interference/core/IndexList.java index a331e86..19c7231 100644 --- a/src/main/java/su/interference/core/IndexList.java +++ b/src/main/java/su/interference/core/IndexList.java @@ -60,7 +60,7 @@ public synchronized void add (String id, Object element) { } public synchronized void add (IndexElementKey key, Object element) { - IndexElement e = new IndexElement(key, element); + IndexElement e = new IndexElement(key, element, false); boolean cnue = true; IndexElementList target = list.get(start); @@ -92,9 +92,9 @@ public synchronized void add (IndexElementKey key, Object element) { addElementList(newlist); prevtg = target; if (newlist.isDivided()) { - e = new IndexElement(newlist.getMaxValue(), new Integer(newlist.getPtr())); + e = new IndexElement(newlist.getMaxValue(), new Integer(newlist.getPtr()), true); } else { - e = new IndexElement(prevtg.getMaxValue(), new Integer(prevtg.getPtr())); + e = new IndexElement(prevtg.getMaxValue(), new Integer(prevtg.getPtr()), true); } if (prevtg.getParent()==0) { //add parent ElementList - always type 2 (node) target = new IndexElementList(2); diff --git a/src/main/java/su/interference/core/Instance.java b/src/main/java/su/interference/core/Instance.java index e49db6e..334bd3c 100644 --- a/src/main/java/su/interference/core/Instance.java +++ b/src/main/java/su/interference/core/Instance.java @@ -400,6 +400,9 @@ private static void registerMetrics() throws Exception { Metrics.register(Metrics.TIMER, "executeQuery"); Metrics.register(Metrics.TIMER, "deallocateQuery"); Metrics.register(Metrics.TIMER, "syncFrames"); + Metrics.register(Metrics.TIMER, "persistGetChunk"); + Metrics.register(Metrics.TIMER, "persistInsertChunk"); + Metrics.register(Metrics.TIMER, "persistInsertIndex"); Metrics.register(Metrics.HISTOGRAM, "recordRCount"); Metrics.register(Metrics.HISTOGRAM, "recordLCount"); Metrics.register(Metrics.HISTOGRAM, "syncQueue"); diff --git a/src/main/java/su/interference/core/ValueSet.java b/src/main/java/su/interference/core/ValueSet.java index d59ef39..801bd2d 100644 --- a/src/main/java/su/interference/core/ValueSet.java +++ b/src/main/java/su/interference/core/ValueSet.java @@ -24,8 +24,6 @@ this software and associated documentation files (the "Software"), to deal in package su.interference.core; -import java.util.Date; - /** * @author Yuriy Glotanov * @since 1.0 @@ -57,26 +55,9 @@ public int compare (Object obj, int thr) { final ValueSet j = (ValueSet)obj; for (int i=0; i (Integer)j.getValueSet()[i]) { return 1; } - } - if (clname.equals("Long")) { - if ((Long)this.vs[i] < (Long)j.getValueSet()[i]) { return -1; } else if ((Long)this.vs[i] > (Long)j.getValueSet()[i]) { return 1; } - } - if (clname.equals("Float")) { - if ((Float)this.vs[i] < (Float)j.getValueSet()[i]) { return -1; } else if ((Float)this.vs[i] > (Float)j.getValueSet()[i]) { return 1; } - } - if (clname.equals("Double")) { - if ((Double)this.vs[i] < (Double)j.getValueSet()[i]) { return -1; } else if ((Double)this.vs[i] > (Double)j.getValueSet()[i]) { return 1; } - } - if (clname.equals("String")) { - int c = (((String)this.vs[i]).compareTo((String)j.getValueSet()[i])); - if (c!=0) { return c; } - } - if (clname.equals("Date")) { - int c = (((Date)this.vs[i]).compareTo((Date)j.getValueSet()[i])); - if (c!=0) { return c; } + final int ct = ((Comparable)this.vs[i]).compareTo(j.getValueSet()[i]); + if (ct != 0) { + return ct; } if (i==thr-1) { break; @@ -90,5 +71,11 @@ public boolean equals (Object obj) { return this.compareTo(j)==0?true:false; } + public int hashCode() { + int hashCode = 1; + for (Object o : vs) + hashCode = 31 * hashCode + (o == null ? 0 : o.hashCode()); + return hashCode; + } } diff --git a/src/main/java/su/interference/persistent/Table.java b/src/main/java/su/interference/persistent/Table.java index 3f7dd79..c130306 100644 --- a/src/main/java/su/interference/persistent/Table.java +++ b/src/main/java/su/interference/persistent/Table.java @@ -118,6 +118,10 @@ public class Table implements DataObject, ResultSet { @Transient private final java.lang.reflect.Field idfield; @Transient + private final String idfieldtype; + @Transient + private final String idfieldgetter; + @Transient private final java.lang.reflect.Field generatedfield; @Transient private final WaitFrame[] lbs; @@ -127,20 +131,18 @@ public class Table implements DataObject, ResultSet { private final AtomicInteger ixFrameCurr = new AtomicInteger(0); @SuppressWarnings("WeakerAccess") @Transient - public Field[] columns; - - public Class getGenericClass() { - return genericClass; - } - - public void setGenericClass(Class genericClass) { - this.genericClass = genericClass; - } + public java.lang.reflect.Field[] fields; + @Transient + public String[] fieldtypes; public Class getSc() { return sc; } + public Class getGenericClass() { + return genericClass; + } + public void setSc(Class sc) { this.sc = sc; } @@ -210,14 +212,6 @@ public long getLastFrameId() { return this.getFileLast()+this.getFrameLast(); } - public Field[] getColumns() { - return columns; - } - - public void setColumns(Field[] columns) { - this.columns = columns; - } - //todo rename to isSystem public boolean isNoTran() throws ClassNotFoundException, MalformedURLException { return this.notran; @@ -424,13 +418,17 @@ private Table getFirstIndexByIdColumn() throws ClassNotFoundException, InternalE return null; } - public java.lang.reflect.Field[] getFields() throws ClassNotFoundException, InternalException, MalformedURLException { + private java.lang.reflect.Field[] getTableFields() throws ClassNotFoundException, MalformedURLException { final Class c = this.getTableClass(); final ArrayList res = new ArrayList(); final java.lang.reflect.Field[] f = c.getDeclaredFields(); for (int i=0; i(); this.maps = new ArrayList(); @@ -989,9 +1036,12 @@ protected DataChunk persist (final Object o, final Session s, final LLT extllt) return null; } + Metrics.get("persistGetChunk").start(); final DataChunk dc = this.getChunkByEntity(o, s); + Metrics.get("persistGetChunk").stop(); if (dc==null) { + Metrics.get("persistInsertChunk").start(); if (this.isNoTran()) { this.ident(o, s, llt); //ident system entities during persist } @@ -1032,8 +1082,11 @@ protected DataChunk persist (final Object o, final Session s, final LLT extllt) //system-only table in-memory indexes this.addIndexValue(nc); bd.release(); + Metrics.get("persistInsertChunk").stop(); + Metrics.get("persistInsertIndex").start(); this.persistIndexes(nc, s, llt); + Metrics.get("persistInsertIndex").stop(); if (extllt == null) { llt.commit(); } @@ -1302,9 +1355,26 @@ public DataChunk getChunkByEntity (Object o, Session s) throws IOException, Invo if (idf == null) { logger.error("No @Id annotated column found for " + this.getName()); } + final MapField mf = this.getMapFieldByColumn(idf.getName()); final IndexField ix = this.getIndexFieldByColumn(idf.getName()); - if (ix!=null) { - final String type = idf.getType().getName(); + final String type = idf.getType().getName(); + if (mf != null) { + if (type.equals("int")||type.equals("java.lang.Integer")) { + final Method z = c.getMethod("get"+idf.getName().substring(0,1).toUpperCase()+idf.getName().substring(1,idf.getName().length()), null); + final int i = (Integer)z.invoke(o, null); + return (DataChunk)mf.getMap().get(i); + } + if (type.equals("long")||type.equals("java.lang.Long")) { + final Method z = c.getMethod("get"+idf.getName().substring(0,1).toUpperCase()+idf.getName().substring(1,idf.getName().length()), null); + final long l = (Long)z.invoke(o, null); + return (DataChunk)mf.getMap().get(l); + } + if (type.equals("java.lang.String")) { + final Method z = c.getMethod("get"+idf.getName().substring(0,1).toUpperCase()+idf.getName().substring(1,idf.getName().length()), null); + final String ss = (String)z.invoke(o, null); + return (DataChunk)mf.getMap().get(ss); + } + } else if (ix != null) { if (type.equals("int")||type.equals("java.lang.Integer")) { final Method z = c.getMethod("get"+idf.getName().substring(0,1).toUpperCase()+idf.getName().substring(1,idf.getName().length()), null); final int i = (Integer)z.invoke(o, null); diff --git a/src/main/java/su/interference/sql/SQLJoinThreadPool.java b/src/main/java/su/interference/sql/SQLJoinThreadPool.java index 3eafce7..8bb0423 100644 --- a/src/main/java/su/interference/sql/SQLJoinThreadPool.java +++ b/src/main/java/su/interference/sql/SQLJoinThreadPool.java @@ -34,7 +34,7 @@ this software and associated documentation files (the "Software"), to deal in public class SQLJoinThreadPool { - private static ExecutorService exec = Executors.newFixedThreadPool(4); + private static ExecutorService exec = Executors.newFixedThreadPool(8); public static ExecutorService getThreadPool() { return exec;