Support for basic indexed strings

This is currently only supported in GEO/BGEO.  If you try to write
Indexedstrs to another file format they will become integers if
that is supported (i.e. PDA, PDB), and not written if integers
aren't supported (PTC).

This is accessible from python as well.

src/lib/Partio.h

@@ -117,6 +117,11 @@ class ParticlesData:public ParticlesInfo
         return static_cast<T*>(dataInternal(attribute,particleIndex));
     }
 
+    /// All indexed strings for an attribute
+    virtual const std::vector<std::string>& indexedStrs(const ParticleAttribute& attr) const=0;
+
+    /// Looks up the index for a given string for a given attribute, returns -1 if not found
+    virtual int lookupIndexedStr(const ParticleAttribute& attribute,const char* str) const=0;
 
     //! Fill the user supplied values array with data corresponding to the given
     //! list of particles. Specify whether or not your indices are sorted. Attributes
@@ -171,7 +176,7 @@ class ParticlesData:public ParticlesInfo
 class ParticlesDataMutable:public ParticlesData
 {
 protected:
-    virtual ~ParticlesDataMutable() {}
+    virtual ~ParticlesDataMutable();
 
 public:
 
@@ -186,6 +191,9 @@ class ParticlesDataMutable:public ParticlesData
         return static_cast<T*>(dataInternal(attribute,particleIndex));
     }
 
+    /// Returns a token for the given string. This allows efficient storage of string data
+    virtual int registerIndexedStr(const ParticleAttribute& attribute,const char* str)=0;
+
     //! Preprocess the data for finding nearest neighbors by sorting into a
     //! KD-Tree. Note: all particle pointers are invalid after this call.
     virtual void sort()=0;

src/lib/PartioAttribute.h

@@ -44,14 +44,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 namespace Partio{
 
 // Particle Types
-enum ParticleAttributeType {NONE=0,VECTOR=1,FLOAT=2,INT=3};
+enum ParticleAttributeType {NONE=0,VECTOR=1,FLOAT=2,INT=3,INDEXEDSTR=4};
 
 template<ParticleAttributeType ENUMTYPE> struct ETYPE_TO_TYPE
-
 {struct UNUSABLE;typedef UNUSABLE TYPE;};
 template<> struct ETYPE_TO_TYPE<VECTOR>{typedef float TYPE;};
 template<> struct ETYPE_TO_TYPE<FLOAT>{typedef float TYPE;};
 template<> struct ETYPE_TO_TYPE<INT>{typedef int TYPE;};
+template<> struct ETYPE_TO_TYPE<INDEXEDSTR>{typedef int TYPE;};
 
 template<class T1,class T2> struct
 IS_SAME{static const bool value=false;};
@@ -66,11 +66,24 @@ typeCheck(const ParticleAttributeType& type)
         case VECTOR: return IS_SAME<typename ETYPE_TO_TYPE<VECTOR>::TYPE,T>::value;
         case FLOAT: return IS_SAME<typename ETYPE_TO_TYPE<FLOAT>::TYPE,T>::value;
         case INT: return IS_SAME<typename ETYPE_TO_TYPE<INT>::TYPE,T>::value;
+        case INDEXEDSTR: return IS_SAME<typename ETYPE_TO_TYPE<INDEXEDSTR>::TYPE,T>::value;
         default: return false; // unknown type
     }
 }
 
-int TypeSize(ParticleAttributeType attrType);
+inline 
+int TypeSize(ParticleAttributeType attrType)
+{
+    switch(attrType){
+        case NONE: return 0;
+        case VECTOR: return sizeof(float);
+        case FLOAT: return sizeof(float);
+        case INT: return sizeof(int);
+        case INDEXEDSTR: return sizeof(int);
+        default: return 0;
+    }
+}
+
 std::string TypeName(ParticleAttributeType attrType);
 
 // Particle Attribute Specifier

src/lib/core/Particle.cpp

@@ -42,6 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 #include <cassert>
 namespace Partio{
 
+ParticlesDataMutable::~ParticlesDataMutable()
+{}
+
 std::string
 TypeName(ParticleAttributeType attrType)
 {
@@ -50,18 +53,7 @@ TypeName(ParticleAttributeType attrType)
         case VECTOR: return "VECTOR";
         case FLOAT: return "FLOAT";
         case INT: return "INT";
-        default: return 0;
-    }
-}
-
-int
-TypeSize(ParticleAttributeType attrType)
-{
-    switch(attrType){
-        case NONE: return 0;
-        case VECTOR: return sizeof(float);
-        case FLOAT: return sizeof(float);
-        case INT: return sizeof(int);
+        case INDEXEDSTR: return "INDEXEDSTR";
         default: return 0;
     }
 }
@@ -117,6 +109,9 @@ print(const ParticlesData* particles)
             case INT:
                 for(int c=0;c<attrs[k].count;c++) std::cout<<accessors[k].raw<int>(it)[c];
                 break;
+            case INDEXEDSTR:
+                for(int c=0;c<attrs[k].count;c++) std::cout<<accessors[k].raw<int>(it)[c];
+                break;
             }
         }
         std::cout<<std::endl;

src/lib/core/ParticleHeaders.cpp

@@ -93,6 +93,25 @@ sort()
     assert(false);
 }
 
+
+int ParticleHeaders::
+registerIndexedStr(const ParticleAttribute& attribute,const char* str)
+{
+    assert(false);
+}
+
+int ParticleHeaders::
+lookupIndexedStr(const ParticleAttribute& attribute,const char* str) const
+{
+    assert(false);
+}
+
+const std::vector<std::string>& ParticleHeaders::
+indexedStrs(const ParticleAttribute& attr) const
+{
+    assert(false);
+}
+
 void ParticleHeaders::
 findPoints(const float bboxMin[3],const float bboxMax[3],std::vector<ParticleIndex>& points) const
 {

src/lib/core/ParticleHeaders.h

@@ -51,10 +51,15 @@ class ParticleHeaders:public ParticlesDataMutable
     bool attributeInfo(const char* attributeName,ParticleAttribute& attribute) const;
     bool attributeInfo(const int attributeInfo,ParticleAttribute& attribute) const;
 
+    int registerIndexedStr(const ParticleAttribute& attribute,const char* str);
+    int lookupIndexedStr(const ParticleAttribute& attribute,const char* str) const;
+    const std::vector<std::string>& indexedStrs(const ParticleAttribute& attr) const;
+
     virtual void dataAsFloat(const ParticleAttribute& attribute,const int indexCount,
         const ParticleIndex* particleIndices,const bool sorted,float* values) const;
 
     void sort();
+
     void findPoints(const float bboxMin[3],const float bboxMax[3],std::vector<ParticleIndex>& points) const;
     float findNPoints(const float center[3],int nPoints,const float maxRadius,
         std::vector<ParticleIndex>& points,std::vector<float>& pointDistancesSquared) const;

src/lib/core/ParticleSimple.cpp

@@ -68,7 +68,6 @@ release() const
     freeCached(const_cast<ParticlesSimple*>(this));
 }
 
-
 int ParticlesSimple::
 numParticles() const
 {
@@ -203,6 +202,7 @@ addAttribute(const char* attribute,ParticleAttributeType type,const int count)
     char* dataPointer=(char*)malloc(allocatedCount*stride);
     attributeData.push_back(dataPointer);
     attributeOffsets.push_back(dataPointer-(char*)0);
+    attributeIndexedStrs.push_back(IndexedStrTable());
 
     return attr;
 }
@@ -305,11 +305,39 @@ dataAsFloat(const ParticleAttribute& attribute,const int indexCount,
     assert(attribute.attributeIndex>=0 && attribute.attributeIndex<(int)attributes.size());
 
     if(attribute.type==FLOAT || attribute.type==VECTOR) dataInternalMultiple(attribute,indexCount,particleIndices,sorted,(char*)values);
-    else if(attribute.type==INT){
+    else if(attribute.type==INT || attribute.type==INDEXEDSTR){
         char* attrrawbase=attributeData[attribute.attributeIndex];
         int* attrbase=(int*)attrrawbase;
         int count=attribute.count;
         for(int i=0;i<indexCount;i++) for(int k=0;k<count;k++) values[i*count+k]=(int)attrbase[particleIndices[i]*count+k];
     }
 }
 
+int ParticlesSimple::
+registerIndexedStr(const ParticleAttribute& attribute,const char* str)
+{
+    IndexedStrTable& table=attributeIndexedStrs[attribute.attributeIndex];
+    std::map<std::string,int>::const_iterator it=table.stringToIndex.find(str);
+    if(it!=table.stringToIndex.end()) return it->second;
+    int newIndex=table.strings.size();
+    table.strings.push_back(str);
+    table.stringToIndex[str]=newIndex;
+    return newIndex;
+}
+
+int ParticlesSimple::
+lookupIndexedStr(Partio::ParticleAttribute const &attribute, char const *str) const
+{
+    const IndexedStrTable& table=attributeIndexedStrs[attribute.attributeIndex];
+    std::map<std::string,int>::const_iterator it=table.stringToIndex.find(str);
+    if(it!=table.stringToIndex.end()) return it->second;
+    return -1;
+}
+
+const std::vector<std::string>& ParticlesSimple::
+indexedStrs(const ParticleAttribute& attr) const
+{
+    const IndexedStrTable& table=attributeIndexedStrs[attr.attributeIndex];
+    return table.strings;
+}
+

src/lib/core/ParticleSimple.h

@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include "Mutex.h"
 #include "../Partio.h"
 
@@ -59,10 +60,11 @@ class ParticlesSimple:public ParticlesDataMutable,
     int numParticles() const;
     bool attributeInfo(const char* attributeName,ParticleAttribute& attribute) const;
     bool attributeInfo(const int attributeInfo,ParticleAttribute& attribute) const;
-
-    virtual void dataAsFloat(const ParticleAttribute& attribute,const int indexCount,
+    void dataAsFloat(const ParticleAttribute& attribute,const int indexCount,
         const ParticleIndex* particleIndices,const bool sorted,float* values) const;
-
+    int registerIndexedStr(const ParticleAttribute& attribute,const char* str);
+    int lookupIndexedStr(const ParticleAttribute& attribute,const char* str) const;
+    const std::vector<std::string>& indexedStrs(const ParticleAttribute& attr) const;
     void sort();
     void findPoints(const float bboxMin[3],const float bboxMax[3],std::vector<ParticleIndex>& points) const;
     float findNPoints(const float center[3],int nPoints,const float maxRadius,
@@ -91,6 +93,11 @@ class ParticlesSimple:public ParticlesDataMutable,
     int allocatedCount;
     std::vector<char*> attributeData; // Inside is data of appropriate type
     std::vector<size_t> attributeOffsets; // Inside is data of appropriate type
+    struct IndexedStrTable{
+        std::map<std::string,int> stringToIndex; // TODO: this should be a hash table unordered_map
+        std::vector<std::string> strings;
+    };
+    std::vector<IndexedStrTable> attributeIndexedStrs;
     std::vector<ParticleAttribute> attributes;
     std::vector<int> attributeStrides;
     std::map<std::string,int> nameToAttribute;

src/lib/io/BGEO.cpp

@@ -127,7 +127,10 @@ ParticlesDataMutable* readBGEO(const char* filename,const bool headersOnly)
             attrOffsets.push_back(particleSize);
             particleSize+=size;
         }else if(houdiniType==4){
-            std::cerr<<"Partio: attr '"<<name<<"' of type index (string) found, treating as integer"<<std::endl;
+            ParticleAttribute attribute=simple->addAttribute(name,INDEXEDSTR,size);
+            attrHandles.push_back(attribute);
+            accessors.push_back(ParticleAccessor(attrHandles.back()));
+            attrOffsets.push_back(particleSize);
             int numIndices=0;
             read<BIGEND>(*input,numIndices);
             for(int ii=0;ii<numIndices;ii++){
@@ -136,12 +139,12 @@ ParticlesDataMutable* readBGEO(const char* filename,const bool headersOnly)
                 char* indexName=new char[indexNameLength+1];;
                 input->read(indexName,indexNameLength);
                 indexName[indexNameLength]=0;
-                std::cerr<<"Partio:    index "<<ii<<" is "<<indexName<<std::endl;
+                int id=simple->registerIndexedStr(attribute,indexName);
+                if(id != ii){
+                    std::cerr<<"Partio: error on read, expected registeerIndexStr to return index "<<ii<<" but got "<<id<<" for string "<<indexName<<std::endl;
+                }
                 delete [] indexName;
             }
-            attrHandles.push_back(simple->addAttribute(name,INT,size));
-            accessors.push_back(ParticleAccessor(attrHandles.back()));
-            attrOffsets.push_back(particleSize);
             particleSize+=size;
         }else if(houdiniType==2){
             std::cerr<<"Partio: found attr of type 'string', aborting"<<std::endl;
@@ -227,18 +230,29 @@ bool writeBGEO(const char* filename,const ParticlesData& p,const bool compressed
             foundPosition=true;
         }else{
             writeHoudiniStr(*output,attr.name);
-            int houdiniType=0;
-            switch(attr.type){
-                case FLOAT: houdiniType=0;break;
-                case INT: houdiniType=1;break;
-                case VECTOR: houdiniType=5;break;
-                case NONE: assert(false);houdiniType=0;break;
-            }
-            unsigned short size=attr.count;
-            write<BIGEND>(*output,size,houdiniType);
-            for(int i=0;i<attr.count;i++){
-                int defaultValue=0;
-                write<BIGEND>(*output,defaultValue);
+            if(attr.type==INDEXEDSTR){
+                int houdiniType=4;
+                unsigned short size=attr.count;
+                const std::vector<std::string>& indexTable=p.indexedStrs(attr);
+                int numIndexes=indexTable.size();
+                write<BIGEND>(*output,size,houdiniType,numIndexes);
+                for(int i=0;i<numIndexes;i++)
+                    writeHoudiniStr(*output,indexTable[i]);
+            }else{
+                int houdiniType=0;
+                switch(attr.type){
+                    case FLOAT: houdiniType=0;break;
+                    case INT: houdiniType=1;break;
+                    case VECTOR: houdiniType=5;break;
+                    case INDEXEDSTR:
+                    case NONE: assert(false);houdiniType=0;break;
+                }
+                unsigned short size=attr.count;
+                write<BIGEND>(*output,size,houdiniType);
+                for(int i=0;i<attr.count;i++){
+                    int defaultValue=0;
+                    write<BIGEND>(*output,defaultValue);
+                }
             }
             attrOffsets.push_back(particleSize);
             particleSize+=attr.count;