diff --git a/cmake/SC_Targets.cmake b/cmake/SC_Targets.cmake
index 8218fad6e..4daa44179 100644
--- a/cmake/SC_Targets.cmake
+++ b/cmake/SC_Targets.cmake
@@ -22,6 +22,9 @@ macro(SC_ADDEXEC execname)
                 message(SEND_ERROR "SC_ADDEXEC usage error - expected STATIC LINK_LIBRARIES targets (${_lib})")
             endif()
         endif()
+        # Executables consume their dependencies but do not publish an
+        # interface to downstream targets.
+        target_link_libraries(${execname} PRIVATE ${_lib})
     endforeach()
     target_link_libraries(${execname} PRIVATE ${${_arg_prefix}_LINK_LIBRARIES})
   endif()
@@ -74,6 +77,11 @@ macro(SC_ADDLIB _addlib_target)
             message(SEND_ERROR "SC_ADDLIB usage error - expected (static) LINK_LIBRARIES targets (${_lib})")
             endif()
         endif()
+        # Schema libraries are consumed directly by generated test programs.
+        # Publish their link requirements explicitly; the legacy unscoped
+        # signature does not reliably provide a transitive interface with
+        # current CMake versions.
+        target_link_libraries(${_addlib_target} PUBLIC ${_lib})
     endforeach()
     target_link_libraries(${_addlib_target} ${_lib})
   endif()
@@ -93,4 +101,3 @@ endmacro()
 # indent-tabs-mode: t
 # End:
 # ex: shiftwidth=2 tabstop=8
-
diff --git a/include/cllazyfile/CMakeLists.txt b/include/cllazyfile/CMakeLists.txt
index 853aa98b9..488e6135d 100644
--- a/include/cllazyfile/CMakeLists.txt
+++ b/include/cllazyfile/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LAZY_HDRS
   p21HeaderSectionReader.h
   lazyDataSectionReader.h
   lazyInstMgr.h
+  lazySupport.h
   lazyTypes.h
   sectionReader.h
   instMgrHelper.h
@@ -24,4 +25,3 @@ install(FILES ${LAZY_HDRS}
 # indent-tabs-mode: t
 # End:
 # ex: shiftwidth=2 tabstop=8
-
diff --git a/include/cllazyfile/headerSectionReader.h b/include/cllazyfile/headerSectionReader.h
index e3a9f5725..2453503b9 100644
--- a/include/cllazyfile/headerSectionReader.h
+++ b/include/cllazyfile/headerSectionReader.h
@@ -27,12 +27,9 @@ class SC_LAZYFILE_EXPORT headerSectionReader: public sectionReader {
         }
 
         virtual ~headerSectionReader() {
-            //FIXME delete each instance?! maybe add to clear, since it iterates over everything already
-            //enum clearHow { rawData, deletePointers }
-            _headerInstances->clear();
+            _headerInstances->clear( true );
 	    delete _headerInstances;
         }
 };
 
 #endif  //HEADERSECTIONREADER_H
-
diff --git a/include/cllazyfile/instMgrHelper.h b/include/cllazyfile/instMgrHelper.h
index 1af698160..881d83ba3 100644
--- a/include/cllazyfile/instMgrHelper.h
+++ b/include/cllazyfile/instMgrHelper.h
@@ -31,7 +31,9 @@ class SC_LAZYFILE_EXPORT mgrNodeHelper: public MgrNodeBase {
             _id = id;
         }
         inline SDAI_Application_instance * GetSTEPentity() {
-            return _lim->loadInstance( _id, true );
+            /* Attribute resolution must not convert a batch-owned cache hit
+             * into a process-lifetime retained instance. */
+            return _lim->loadInstance( _id, true, false );
         }
 };
 
@@ -58,4 +60,3 @@ class SC_LAZYFILE_EXPORT instMgrAdapter: public InstMgrBase {
 
 
 #endif //INSTMGRHELPER_H
-
diff --git a/include/cllazyfile/judyLArray.h b/include/cllazyfile/judyLArray.h
index b78dc0bb9..877420549 100644
--- a/include/cllazyfile/judyLArray.h
+++ b/include/cllazyfile/judyLArray.h
@@ -159,8 +159,8 @@ class judyLArray {
          * getLastValue() will return the entry before the one that was deleted
          * \sa isEmpty()
          */
-        bool removeEntry( JudyKey * key ) {
-            if( judy_slot( _judyarray, key, _depth * JUDY_key_size ) ) {
+        bool removeEntry( JudyKey key ) {
+            if( judy_slot( _judyarray, reinterpret_cast<const unsigned char *>( &key ), _depth * JUDY_key_size ) ) {
                 _lastSlot = ( JudyValue * ) judy_del( _judyarray );
                 return true;
             } else {
diff --git a/include/cllazyfile/lazyFileReader.h b/include/cllazyfile/lazyFileReader.h
index 9a5fccb5e..97b53f1ea 100644
--- a/include/cllazyfile/lazyFileReader.h
+++ b/include/cllazyfile/lazyFileReader.h
@@ -37,6 +37,8 @@ class SC_LAZYFILE_EXPORT lazyFileReader {
 #endif
         fileTypeEnum _fileType;
         fileID _fileID;
+        lazyFileOffset _fileSize;
+        bool _valid;
 
         void initP21();
 
@@ -48,6 +50,12 @@ class SC_LAZYFILE_EXPORT lazyFileReader {
         fileID ID() const {
             return _fileID;
         }
+        lazyFileOffset fileSize() const {
+            return _fileSize;
+        }
+        bool valid() const {
+            return _valid;
+        }
         instancesLoaded_t * getHeaderInstances();
 
         lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid );
@@ -64,4 +72,3 @@ class SC_LAZYFILE_EXPORT lazyFileReader {
 };
 
 #endif //LAZYFILEREADER_H
-
diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h
index e3445660a..aac042800 100644
--- a/include/cllazyfile/lazyInstMgr.h
+++ b/include/cllazyfile/lazyInstMgr.h
@@ -1,13 +1,17 @@
 #ifndef LAZYINSTMGR_H
 #define LAZYINSTMGR_H
 
+#include <cctype>
 #include <map>
+#include <set>
 #include <string>
+#include <vector>
 #include <assert.h>
 
 #include "cllazyfile/lazyDataSectionReader.h"
 #include "cllazyfile/lazyFileReader.h"
 #include "cllazyfile/lazyTypes.h"
+#include "cllazyfile/lazySupport.h"
 
 #include "clstepcore/Registry.h"
 #include "sc_export.h"
@@ -19,6 +23,7 @@
 
 class Registry;
 class instMgrAdapter;
+class lazyRefs;
 
 class SC_LAZYFILE_EXPORT lazyInstMgr {
     protected:
@@ -61,15 +66,50 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
 
         lazyFileReaderVec_t _files;
 
+        /** All indexed DATA instance IDs in deterministic file order. */
+        instanceRefs _allInstances;
+
         Registry * _headerRegistry, * _mainRegistry;
+        bool _ownsMainRegistry;
         ErrorDescriptor * _errors;
 
         unsigned long _lazyInstanceCount, _loadedInstanceCount;
+        uint64_t _cacheHighWater, _cacheHits, _cacheMisses, _materializations, _evictions;
+        uint64_t _activeBatches;
+        uint64_t _residentSourceBytes, _sourceBytesHighWater;
         int _longestTypeNameLen;
         std::string _longestTypeName;
 
+        std::map<instanceID, size_t> _pinCounts;
+        std::map<instanceID, uint64_t> _instanceSourceBytes;
+        std::map<std::string, std::string> _materializationTypeAliases;
+        std::set<instanceID> _batchOwnedInstances;
+        std::set<instanceID> _permanentlyLoadedInstances;
+        std::set<instanceID> _instancesLoading;
+        std::set<instanceID> _deferredInverseInstances;
+        size_t _batchLoadDepth;
+
+        LazyProgressCallback _progressCallback;
+        LazyCancellationCallback _cancellationCallback;
+        LazyDiagnosticCallback _diagnosticCallback;
+        std::map<std::string, uint64_t> _diagnosticCounts;
+        bool _cancelled;
+
         instMgrAdapter * _ima;
 
+        friend class LazyInstanceBatch;
+        friend class lazyRefs;
+        void releaseBatch( const std::vector<instanceID> & instances );
+        void resolveDeferredInverses();
+        bool isMaterializing( instanceID id ) const {
+            return _instancesLoading.count( id ) != 0;
+        }
+        void deferInverseResolution( instanceID id ) {
+            _deferredInverseInstances.insert( id );
+        }
+        SDAI_Application_instance * cachedInstance( instanceID id );
+        std::vector<instanceID> dependencyClosure( const std::vector<instanceID> & roots );
+
 #ifdef _MSC_VER
 #pragma warning( pop )
 #endif
@@ -77,13 +117,34 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
     public:
         lazyInstMgr();
         ~lazyInstMgr();
-        void openFile( std::string fname );
+        bool openFile( std::string fname );
 
         void addLazyInstance( namedLazyInstance inst );
         InstMgrBase * getAdapter() {
             return ( InstMgrBase * ) _ima;
         }
 
+        /** Register an explicit source-keyword substitution used only when
+         * materializing an SDAI object.  The lazy type index retains the
+         * original Part 21 keyword.  This is intended for known,
+         * attribute-compatible exporter aliases, not arbitrary recovery. */
+        void setMaterializationTypeAlias( std::string source, const std::string & target ) {
+            for( std::string::iterator c = source.begin(); c != source.end(); ++c ) {
+                *c = static_cast<char>( toupper( static_cast<unsigned char>( *c ) ) );
+            }
+            _materializationTypeAliases[source] = target;
+        }
+
+        std::string materializationType( std::string source ) const {
+            std::string key = source;
+            for( std::string::iterator c = key.begin(); c != key.end(); ++c ) {
+                *c = static_cast<char>( toupper( static_cast<unsigned char>( *c ) ) );
+            }
+            std::map<std::string, std::string>::const_iterator alias =
+                _materializationTypeAliases.find( key );
+            return alias == _materializationTypeAliases.end() ? source : alias->second;
+        }
+
         instanceRefs_t * getFwdRefs() {
             return & _fwdInstanceRefs;
         }
@@ -91,19 +152,29 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
         instanceRefs_t * getRevRefs() {
             return & _revInstanceRefs;
         }
+        LazyInstanceIdView instancesByType( std::string type, bool caseSensitive = false );
+        LazyInstanceIdView allInstances() const {
+            return LazyInstanceIdView( &_allInstances );
+        }
+        LazyInstanceIdView forwardReferences( instanceID id );
+        LazyInstanceIdView reverseReferences( instanceID id );
+        /** Copy the exact indexed source record for an instance.  Returns an
+         * empty string when the ID is missing or ambiguous. */
+        std::string sourceRecord( instanceID id );
+
         /// returns a vector containing the instances that match `type`
         instanceTypes_t::cvector * getInstances( std::string type, bool caseSensitive = false ) { /*const*/
             if( !caseSensitive ) {
                 std::string::iterator it = type.begin();
                 for( ; it != type.end(); ++it ) {
-                    *it = toupper( *it );
+                    *it = static_cast<char>( toupper( static_cast<unsigned char>( *it ) ) );
                 }
             }
             return _instanceTypes->find( type.c_str() );
         }
         /// get the number of instances of a certain type
-        unsigned int countInstances( std::string type ) {
-            instanceTypes_t::cvector * v = _instanceTypes->find( type.c_str() );
+        unsigned int countInstances( std::string type, bool caseSensitive = false ) {
+            instanceTypes_t::cvector * v = getInstances( type, caseSensitive );
             if( !v ) {
                 return 0;
             }
@@ -123,6 +194,28 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
             return _loadedInstanceCount;
         }
 
+        LazyCacheStatistics cacheStatistics() const;
+
+        void setProgressCallback( const LazyProgressCallback & callback ) {
+            _progressCallback = callback;
+        }
+        void setCancellationCallback( const LazyCancellationCallback & callback ) {
+            _cancellationCallback = callback;
+        }
+        void setDiagnosticCallback( const LazyDiagnosticCallback & callback ) {
+            _diagnosticCallback = callback;
+        }
+        bool cancelled() const {
+            return _cancelled;
+        }
+        void observeScan( fileID file, lazyFileOffset offset, lazyFileOffset fileSize );
+        void emitDiagnostic( LazyDiagnostic diagnostic );
+        uint64_t diagnosticCount( const std::string & key ) const;
+        const std::map<std::string, uint64_t> & diagnosticCounts() const {
+            return _diagnosticCounts;
+        }
+        void validateReferences();
+
         /// get the number of data sections that have been identified
         unsigned int countDataSections() {
             return _dataSections.size();
@@ -130,7 +223,9 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
 
         ///builds the registry using the given initFunct
         const Registry * initRegistry( CF_init initFunct ) {
-            setRegistry( new Registry( initFunct ) );
+            assert( _mainRegistry == 0 );
+            _mainRegistry = new Registry( initFunct );
+            _ownsMainRegistry = true;
             return _mainRegistry;
         }
 
@@ -138,6 +233,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
         void setRegistry( Registry * reg ) {
             assert( _mainRegistry == 0 );
             _mainRegistry = reg;
+            _ownsMainRegistry = false;
         }
 
         const Registry * getHeaderRegistry() const {
@@ -165,8 +261,12 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
         /** returns a pointer to an instance, loading it if necessary.
          * \param id the instance number to look for
          * \param reSeek if true, reset file position to current position when done. only necessary when loading an instance with dependencies; excessive use will cause a performance hit
+         * \param promoteCached if true, a cached instance requested outside a batch is retained for legacy callers
          */
-        SDAI_Application_instance * loadInstance( instanceID id, bool reSeek = false );
+        SDAI_Application_instance * loadInstance( instanceID id, bool reSeek = false, bool promoteCached = true );
+
+        LazyInstanceBatch loadBatch( instanceID root );
+        LazyInstanceBatch loadBatch( const std::vector<instanceID> & roots );
 
         //list all instances that one instance depends on (recursive)
         instanceSet * instanceDependencies( instanceID id );
@@ -183,11 +283,8 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
                     std::cerr << "Error at " << __FILE__ << ":" << __LINE__ << " - multiple instances (" << cv->size() << ") with one instanceID (" << id << ") not supported yet." << std::endl;
                     return 0;
                 }
-                positionAndSection ps = cv->at( 0 );
-                //extract p, s, call
-                long int off = ps & 0xFFFFFFFFFFFFULL;
-                sectionID sid = ps >> 48;
-                return _dataSections[sid]->getType( off );
+                instancePosition pos = cv->at( 0 );
+                return _dataSections[pos.section]->getType( pos.begin );
             }
             std::cerr << "Error at " << __FILE__ << ":" << __LINE__ << " - instanceID " << id << " not found." << std::endl;
             return 0;
@@ -218,4 +315,3 @@ class SC_LAZYFILE_EXPORT lazyInstMgr {
 };
 
 #endif //LAZYINSTMGR_H
-
diff --git a/include/cllazyfile/lazyP21DataSectionReader.h b/include/cllazyfile/lazyP21DataSectionReader.h
index c14376fc0..f24c2802e 100644
--- a/include/cllazyfile/lazyP21DataSectionReader.h
+++ b/include/cllazyfile/lazyP21DataSectionReader.h
@@ -7,12 +7,13 @@
 
 class SC_LAZYFILE_EXPORT lazyP21DataSectionReader: public lazyDataSectionReader {
     protected:
+        /** Index the instances in an Edition 1 SCOPE and leave the stream at
+         * the owning entity's record. */
+        bool indexScope();
     public:
         lazyP21DataSectionReader( lazyFileReader * parent, std::ifstream & file, std::streampos start, sectionID sid );
 
-        void findSectionStart() {
-            _sectionStart = findNormalString( "DATA", true );
-        }
+        void findSectionStart();
         /** gets information (start, end, name, etc) about the next
          * instance in the file and returns it in a namedLazyInstance
          * \sa p21HeaderSectionReader::nextInstance()
@@ -22,4 +23,3 @@ class SC_LAZYFILE_EXPORT lazyP21DataSectionReader: public lazyDataSectionReader
 };
 
 #endif //LAZYP21DATASECTIONREADER_H
-
diff --git a/include/cllazyfile/lazySupport.h b/include/cllazyfile/lazySupport.h
new file mode 100644
index 000000000..79a04018f
--- /dev/null
+++ b/include/cllazyfile/lazySupport.h
@@ -0,0 +1,131 @@
+#ifndef LAZYSUPPORT_H
+#define LAZYSUPPORT_H
+
+#include <cstddef>
+#include <functional>
+#include <iosfwd>
+#include <string>
+#include <vector>
+
+#include "cllazyfile/lazyTypes.h"
+#include "sc_export.h"
+
+class lazyInstMgr;
+class SDAI_Application_instance;
+
+enum LazyDiagnosticSeverity {
+    LAZY_DIAGNOSTIC_INFO,
+    LAZY_DIAGNOSTIC_WARNING,
+    LAZY_DIAGNOSTIC_ERROR,
+    LAZY_DIAGNOSTIC_FATAL
+};
+
+struct SC_LAZYFILE_EXPORT LazyDiagnostic {
+    LazyDiagnosticSeverity severity;
+    instanceID entity;
+    std::string type;
+    lazyFileOffset offset;
+    uint64_t line;
+    std::string attribute;
+    std::string message;
+    uint64_t occurrences;
+
+    LazyDiagnostic(): severity( LAZY_DIAGNOSTIC_INFO ), entity( 0 ), offset( 0 ),
+        line( 0 ), occurrences( 1 ) {}
+};
+
+struct SC_LAZYFILE_EXPORT LazyScanProgress {
+    fileID file;
+    lazyFileOffset offset;
+    lazyFileOffset fileSize;
+    uint64_t instancesScanned;
+
+    LazyScanProgress(): file( 0 ), offset( 0 ), fileSize( 0 ), instancesScanned( 0 ) {}
+};
+
+struct SC_LAZYFILE_EXPORT LazyCacheStatistics {
+    uint64_t instancesScanned;
+    uint64_t instancesLoaded;
+    uint64_t instancesPinned;
+    uint64_t cacheHighWater;
+    uint64_t cacheHits;
+    uint64_t cacheMisses;
+    uint64_t materializations;
+    uint64_t evictions;
+    uint64_t activeBatches;
+    uint64_t dataSections;
+    uint64_t residentSourceBytes;
+    uint64_t sourceBytesHighWater;
+    bool cancelled;
+
+    LazyCacheStatistics(): instancesScanned( 0 ), instancesLoaded( 0 ), instancesPinned( 0 ),
+        cacheHighWater( 0 ), cacheHits( 0 ), cacheMisses( 0 ), materializations( 0 ),
+        evictions( 0 ), activeBatches( 0 ), dataSections( 0 ), residentSourceBytes( 0 ),
+        sourceBytesHighWater( 0 ), cancelled( false ) {}
+};
+
+typedef std::function<void ( const LazyScanProgress & )> LazyProgressCallback;
+typedef std::function<bool ()> LazyCancellationCallback;
+typedef std::function<void ( const LazyDiagnostic & )> LazyDiagnosticCallback;
+
+/** A non-owning, zero-copy view of an indexed instance-ID vector.
+ * The view remains valid until another file is scanned by the manager.
+ */
+class SC_LAZYFILE_EXPORT LazyInstanceIdView {
+    public:
+        typedef instanceRefs::const_iterator const_iterator;
+
+        LazyInstanceIdView(): _ids( 0 ) {}
+        explicit LazyInstanceIdView( const instanceRefs * ids ): _ids( ids ) {}
+
+        const_iterator begin() const;
+        const_iterator end() const;
+        size_t size() const;
+        bool empty() const;
+        instanceID operator[]( size_t index ) const;
+
+    private:
+        const instanceRefs * _ids;
+};
+
+/** A dependency closure materialized and pinned for a bounded operation.
+ * STEPcode parsing remains single-threaded.  Detach immutable conversion data
+ * before releasing the batch or passing work to another thread.
+ */
+class SC_LAZYFILE_EXPORT LazyInstanceBatch {
+    public:
+        LazyInstanceBatch();
+        ~LazyInstanceBatch();
+        LazyInstanceBatch( LazyInstanceBatch && other );
+        LazyInstanceBatch & operator=( LazyInstanceBatch && other );
+
+        LazyInstanceBatch( const LazyInstanceBatch & ) = delete;
+        LazyInstanceBatch & operator=( const LazyInstanceBatch & ) = delete;
+
+        bool valid() const;
+        void release();
+        const std::vector<instanceID> & roots() const;
+        const std::vector<instanceID> & instances() const;
+        SDAI_Application_instance * get( instanceID id ) const;
+
+    private:
+        friend class lazyInstMgr;
+        LazyInstanceBatch( lazyInstMgr * manager, const std::vector<instanceID> & roots,
+            const std::vector<instanceID> & instances );
+
+        lazyInstMgr * _manager;
+        std::vector<instanceID> _roots;
+        std::vector<instanceID> _instances;
+};
+
+/** Adapter preserving a stream-oriented diagnostic presentation. */
+class SC_LAZYFILE_EXPORT LazyTextDiagnosticAdapter {
+    public:
+        explicit LazyTextDiagnosticAdapter( std::ostream & stream );
+        void operator()( const LazyDiagnostic & diagnostic ) const;
+
+    private:
+        std::ostream * _stream;
+};
+
+#endif // LAZYSUPPORT_H
diff --git a/include/cllazyfile/lazyTypes.h b/include/cllazyfile/lazyTypes.h
index f1fdbd782..5bcbdcf30 100644
--- a/include/cllazyfile/lazyTypes.h
+++ b/include/cllazyfile/lazyTypes.h
@@ -6,6 +6,7 @@
 #include <iostream>
 #include <vector>
 #include <set>
+#include <string>
 
 #ifdef HAVE_STDINT_H
 #include <stdint.h>
@@ -29,6 +30,7 @@ enum fileTypeEnum { Part21, Part28 };
 // enum loadingEnum { immediate, lazy };
 
 typedef uint64_t instanceID;  ///< the number assigned to an instance in the file
+typedef uint64_t lazyFileOffset; ///< byte offset in an exchange file
 typedef uint16_t sectionID;   ///< globally unique index of a sectionReader in a sectionReaderVec_t
 typedef uint16_t fileID;      ///< the index of a lazyFileReader in a lazyFileReaderVec_t. Can be inferred from a sectionID
 
@@ -42,7 +44,10 @@ typedef uint16_t fileID;      ///< the index of a lazyFileReader in a lazyFileRe
  * section = ( ps >> 48 );
  * TODO: Also 8 bits of flags? Would allow files of 2^40 bytes, or ~1TB.
  */
-typedef uint64_t positionAndSection;
+typedef struct {
+    lazyFileOffset begin;
+    sectionID section;
+} instancePosition;
 
 typedef std::vector< instanceID > instanceRefs;
 
@@ -55,7 +60,8 @@ typedef std::set< instanceID > instanceSet;
  * situations, so the information should be kept up-to-date.
  */
 typedef struct {
-    long begin; ///< this is the result of tellg() before reading the instanceID; there may be whitespace or comments, but nothing else.
+    lazyFileOffset begin; ///< byte offset before the instance ID; whitespace or comments may precede it.
+    lazyFileOffset end; ///< byte offset immediately after the terminating semicolon.
     instanceID instance;
     sectionID section;
     /* bool modified; */ /* this will be useful when writing instances - if an instance is
@@ -67,6 +73,7 @@ typedef struct {
     lazyInstanceLoc loc;
     const char * name;
     instanceRefs * refs;
+    std::vector<std::string> * componentTypes; ///< compositional types for a complex instance
 } namedLazyInstance;
 
 // instanceRefs - map between an instanceID and instances that refer to it
@@ -80,7 +87,7 @@ typedef judyLArray< instanceID, SDAI_Application_instance * > instancesLoaded_t;
 
 // instanceStreamPos - map instance id to a streampos and data section
 // there could be multiple instances with the same ID, but in different files (or different sections of the same file?)
-typedef judyL2Array< instanceID, positionAndSection > instanceStreamPos_t;
+typedef judyL2Array< instanceID, instancePosition > instanceStreamPos_t;
 
 
 // data sections
@@ -93,4 +100,3 @@ typedef std::vector< lazyFileReader * > lazyFileReaderVec_t;
 // NOTE not useful? typedef std::vector< lazyInstance > lazyInstanceVec_t;
 
 #endif //LAZYTYPES_H
-
diff --git a/include/cllazyfile/sectionReader.h b/include/cllazyfile/sectionReader.h
index 74f1fa5bf..4665bb765 100644
--- a/include/cllazyfile/sectionReader.h
+++ b/include/cllazyfile/sectionReader.h
@@ -46,12 +46,31 @@ class SC_LAZYFILE_EXPORT sectionReader {
          */
         std::streampos findNormalString( const std::string & str, bool semicolon = false );
 
+        /** Skip from immediately after a comment's opening slash and star
+         * through its closing star and slash.  Part 21 comments are opaque:
+         * apostrophes and string-control text inside them have no syntax.
+         */
+        bool skipComment();
+
+        /** Skip whitespace and Part 21 comments, leaving the next syntactic
+         * token unread. */
+        bool skipTokenSeparators();
+
+        /** Skip an Edition 1 SCOPE construct, including nested scopes and its
+         * optional export list.  The stream must initially point at '&' and
+         * is left at the owning entity's record. */
+        bool skipScope();
+
+        /** Skip an optional scope export list at the current stream
+         * position. */
+        bool skipScopeExportList();
+
         /** Get a keyword ending with one of delimiters.
          */
         const char * getDelimitedKeyword( const char * delimiters );
 
         /** Seek to the end of the current instance */
-        std::streampos seekInstanceEnd( instanceRefs ** refs );
+        std::streampos seekInstanceEnd( instanceRefs ** refs, std::vector<std::string> * componentTypes = 0 );
 
         /// operator>> is very slow?!
         inline void skipWS() {
@@ -63,9 +82,15 @@ class SC_LAZYFILE_EXPORT sectionReader {
         STEPcomplex * CreateSubSuperInstance( const Registry * reg, instanceID fileid, Severity & sev );
 
     public:
-        SDAI_Application_instance * getRealInstance( const Registry * reg, long int begin, instanceID instance,
+        SDAI_Application_instance * getRealInstance( const Registry * reg, lazyFileOffset begin, instanceID instance,
                 const std::string & typeName = "", const std::string & schName = "", bool header = false );
 
+        /** Return one indexed source record without materializing it.  The
+         * stream position is restored before returning.  This is intended
+         * for lightweight adapters which can detach scalar/reference data
+         * directly from Part 21 source. */
+        std::string sourceRecord( lazyFileOffset begin, uint64_t length );
+
         sectionID ID() const {
             return _sectionID;
         }
@@ -91,13 +116,15 @@ class SC_LAZYFILE_EXPORT sectionReader {
         /** returns the type string for an instance, read straight from the file
          * if this function changes, probably need to change nextInstance() as well
          * don't check errors - they would have been encountered during the initial file scan, and the file is still open so it can't have been modified */
-        const char * getType( long int offset ) {
+        const char * getType( lazyFileOffset offset ) {
             if( offset <= 0 ) {
                 return 0;
             }
-            _file.seekg( offset );
+            _file.seekg( static_cast<std::streamoff>( offset ) );
             readInstanceNumber();
-            skipWS();
+            if( !skipTokenSeparators() ) return 0;
+            if( _file.peek() == '&' && !skipScope() ) return 0;
+            if( !skipTokenSeparators() ) return 0;
             return getDelimitedKeyword( ";( /\\" );
         }
 
@@ -112,4 +139,3 @@ class SC_LAZYFILE_EXPORT sectionReader {
 };
 
 #endif //SECTIONREADER_H
-
diff --git a/src/cllazyfile/CMakeLists.txt b/src/cllazyfile/CMakeLists.txt
index 83b2d7137..b45c40462 100644
--- a/src/cllazyfile/CMakeLists.txt
+++ b/src/cllazyfile/CMakeLists.txt
@@ -2,6 +2,7 @@ set(LAZY_SRCS
   lazyDataSectionReader.cc
   lazyFileReader.cc
   lazyInstMgr.cc
+  lazySupport.cc
   p21HeaderSectionReader.cc
   sectionReader.cc
   lazyP21DataSectionReader.cc
@@ -42,3 +43,17 @@ endif()
 # End:
 # ex: shiftwidth=2 tabstop=8
 
+if(SC_ENABLE_TESTING)
+  if(BUILD_SHARED_LIBS)
+    set(_lazy_index_libs steplazyfile stepeditor stepcore stepdai steputils)
+  else()
+    set(_lazy_index_libs steplazyfile-static stepeditor-static
+      stepcore-static stepdai-static steputils-static)
+  endif()
+  SC_ADDEXEC(lazy_index_test SOURCES lazy_index_test.cc
+    LINK_LIBRARIES ${_lazy_index_libs} NO_INSTALL)
+  add_test(NAME lazy_index
+    COMMAND lazy_index_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_index.stp
+      ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_scope.stp)
+endif()
diff --git a/src/cllazyfile/lazyDataSectionReader.cc b/src/cllazyfile/lazyDataSectionReader.cc
index 6de006fe4..3fbb802f2 100644
--- a/src/cllazyfile/lazyDataSectionReader.cc
+++ b/src/cllazyfile/lazyDataSectionReader.cc
@@ -6,7 +6,7 @@
 lazyDataSectionReader::lazyDataSectionReader( lazyFileReader * parent, std::ifstream & file,
         std::streampos start, sectionID sid ):
     sectionReader( parent, file, start, sid ) {
-    _sectionIdentifier = ""; //FIXME set _sectionIdentifier from the data section identifier (2002 rev of Part 21), if present
+    _sectionIdentifier = ""; //FIXME retain the data section identifier (2002 revision of Part 21)
     _error = false;
+    _completelyLoaded = false;
 }
-
diff --git a/src/cllazyfile/lazyFileReader.cc b/src/cllazyfile/lazyFileReader.cc
index fed35c6f5..3705b5517 100644
--- a/src/cllazyfile/lazyFileReader.cc
+++ b/src/cllazyfile/lazyFileReader.cc
@@ -19,20 +19,44 @@ void lazyFileReader::initP21() {
         }
         _parent->registerDataSection( r );
 
+        if( _parent->cancelled() ) break;
+
         //check for new data section (DATA) or end of file (END-ISO-10303-21;)
-        while( isspace( _file.peek() ) && _file.good() ) {
-            _file.ignore( 1 );
+        for( ;; ) {
+            while( isspace( _file.peek() ) && _file.good() ) _file.ignore( 1 );
+            if( _file.peek() != '/' ) break;
+            std::streampos comment = _file.tellg();
+            _file.get();
+            if( _file.peek() != '*' ) {
+                _file.seekg( comment );
+                break;
+            }
+            _file.get();
+            int previous = 0;
+            int current = 0;
+            while( _file.good() ) {
+                current = _file.get();
+                if( previous == '*' && current == '/' ) break;
+                previous = current;
+            }
         }
+        std::streampos nextSection = _file.tellg();
         if( needKW( "END-ISO-10303-21;" ) ) {
             break;
-        } else if( !needKW( "DATA" ) ) {
+        }
+        _file.clear();
+        _file.seekg( nextSection );
+        if( !needKW( "DATA" ) ) {
             std::cerr << "Corrupted file - did not find new data section (\"DATA\") or end of file (\"END-ISO-10303-21;\") at offset " << _file.tellg() << std::endl;
             break;
         }
+        _file.clear();
+        _file.seekg( nextSection );
     }
 }
 
 bool lazyFileReader::needKW( const char * kw ) {
+    std::streampos start = _file.tellg();
     const char * c = kw;
     bool found = true;
     while( *c ) {
@@ -42,6 +66,10 @@ bool lazyFileReader::needKW( const char * kw ) {
         }
         c++;
     }
+    if( !found ) {
+        _file.clear();
+        _file.seekg( start );
+    }
     return found;
 }
 
@@ -49,11 +77,25 @@ instancesLoaded_t * lazyFileReader::getHeaderInstances() {
     return _header->getInstances();
 }
 
-lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ): _fileName( fname ), _parent( i ), _fileID( fid ) {
+lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ): _fileName( fname ), _parent( i ),
+        _header( 0 ), _fileID( fid ), _fileSize( 0 ), _valid( false ) {
     _file.open( _fileName.c_str(), std::ios::binary );
     _file.imbue( std::locale::classic() );
     _file.unsetf( std::ios_base::skipws );
-    assert( _file.is_open() && _file.good() );
+    if( !_file.is_open() || !_file.good() ) {
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_FATAL;
+        diagnostic.message = "unable to open exchange file: " + _fileName;
+        _parent->emitDiagnostic( diagnostic );
+        return;
+    }
+
+    _file.seekg( 0, std::ios::end );
+    std::streampos end = _file.tellg();
+    if( end != std::streampos( -1 ) ) {
+        _fileSize = static_cast<lazyFileOffset>( static_cast<std::streamoff>( end ) );
+    }
+    _file.seekg( 0, std::ios::beg );
 
     detectType();
     switch( _fileType ) {
@@ -67,9 +109,9 @@ lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid )
             std::cerr << "Reached default case, " << __FILE__ << ":" << __LINE__ << std::endl;
             abort();
     }
+    _valid = true;
 }
 
 lazyFileReader::~lazyFileReader() {
     delete _header;
 }
-
diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc
index 65f3e7558..ad42458f7 100644
--- a/src/cllazyfile/lazyInstMgr.cc
+++ b/src/cllazyfile/lazyInstMgr.cc
@@ -8,22 +8,35 @@
 
 #include "clstepcore/sdaiApplication_instance.h"
 
+#include <algorithm>
+#include <climits>
+#include <sstream>
+
 lazyInstMgr::lazyInstMgr() {
     _headerRegistry = new Registry( HeaderSchemaInit );
     _instanceTypes = new instanceTypes_t( 255 ); //NOTE arbitrary max of 255 chars for a type name
     _lazyInstanceCount = 0;
     _loadedInstanceCount = 0;
+    _cacheHighWater = 0;
+    _cacheHits = 0;
+    _cacheMisses = 0;
+    _materializations = 0;
+    _evictions = 0;
+    _activeBatches = 0;
+    _residentSourceBytes = 0;
+    _sourceBytesHighWater = 0;
+    _batchLoadDepth = 0;
+    _cancelled = false;
     _longestTypeNameLen = 0;
     _mainRegistry = 0;
+    _ownsMainRegistry = false;
     _errors = new ErrorDescriptor();
     _ima = new instMgrAdapter( this );
 }
 
 lazyInstMgr::~lazyInstMgr() {
-    delete _headerRegistry;
-    delete _errors;
-    delete _ima;
-    //loop over files, sections, instances; delete header instances
+    // Keep registries and the adapter alive until their instances are gone.
+    _instancesLoaded.clear( true );
     lazyFileReaderVec_t::iterator fit = _files.begin();
     for( ; fit != _files.end(); ++fit ) {
         delete *fit;
@@ -32,8 +45,12 @@ lazyInstMgr::~lazyInstMgr() {
     for( ; sit != _dataSections.end(); ++sit ) {
         delete *sit;
     }
-    _instancesLoaded.clear();
     _instanceStreamPos.clear();
+    delete _instanceTypes;
+    delete _ima;
+    if( _ownsMainRegistry ) delete _mainRegistry;
+    delete _headerRegistry;
+    delete _errors;
 }
 
 sectionID lazyInstMgr::registerDataSection( lazyDataSectionReader * sreader ) {
@@ -50,18 +67,35 @@ void lazyInstMgr::addLazyInstance( namedLazyInstance inst ) {
         _longestTypeName = inst.name;
     }
     _instanceTypes->insert( inst.name, inst.loc.instance );
-    /* store 16 bits of section id and 48 of instance offset into one 64-bit int
-    ** TODO: check and warn if anything is lost (in calling code?)
-    ** does 32bit need anything special?
-    **
-    ** create conversion class?
-    **  could then initialize conversion object with number of bits
-    **  also a good place to check for data loss
-    */
-    positionAndSection ps = inst.loc.section;
-    ps <<= 48;
-    ps |= ( inst.loc.begin & 0xFFFFFFFFFFFFULL );
-    _instanceStreamPos.insert( inst.loc.instance, ps );
+    if( inst.componentTypes ) {
+        std::vector<std::string>::const_iterator type = inst.componentTypes->begin();
+        for( ; type != inst.componentTypes->end(); ++type ) {
+            _instanceTypes->insert( type->c_str(), inst.loc.instance );
+            if( static_cast<int>( type->size() ) > _longestTypeNameLen ) {
+                _longestTypeNameLen = type->size();
+                _longestTypeName = *type;
+            }
+        }
+        delete inst.componentTypes;
+    }
+    const bool duplicate = _instanceStreamPos.find( inst.loc.instance ) != 0;
+    if( !duplicate ) _allInstances.push_back( inst.loc.instance );
+    instancePosition pos;
+    pos.begin = inst.loc.begin;
+    pos.section = inst.loc.section;
+    if( duplicate ) {
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_WARNING;
+        diagnostic.entity = inst.loc.instance;
+        diagnostic.type = inst.name;
+        diagnostic.offset = inst.loc.begin;
+        diagnostic.message = "duplicate instance identifier";
+        emitDiagnostic( diagnostic );
+    }
+    _instanceStreamPos.insert( inst.loc.instance, pos );
+    if( !duplicate && inst.loc.end >= inst.loc.begin ) {
+        _instanceSourceBytes[inst.loc.instance] = inst.loc.end - inst.loc.begin;
+    }
 
     if( inst.refs ) {
         if( inst.refs->size() > 0 ) {
@@ -72,10 +106,96 @@ void lazyInstMgr::addLazyInstance( namedLazyInstance inst ) {
                 //reverse refs
                 _revInstanceRefs.insert( *it, inst.loc.instance );
             }
-        } else {
-            delete inst.refs;
+        }
+        delete inst.refs;
+    }
+}
+
+LazyInstanceIdView lazyInstMgr::instancesByType( std::string type, bool caseSensitive ) {
+    if( !caseSensitive ) {
+        std::string::iterator it = type.begin();
+        for( ; it != type.end(); ++it ) {
+            *it = static_cast<char>( toupper( static_cast<unsigned char>( *it ) ) );
         }
     }
+    return LazyInstanceIdView( _instanceTypes->find( type.c_str() ) );
+}
+
+LazyInstanceIdView lazyInstMgr::forwardReferences( instanceID id ) {
+    return LazyInstanceIdView( _fwdInstanceRefs.find( id ) );
+}
+
+LazyInstanceIdView lazyInstMgr::reverseReferences( instanceID id ) {
+    return LazyInstanceIdView( _revInstanceRefs.find( id ) );
+}
+
+std::string lazyInstMgr::sourceRecord( instanceID id ) {
+    instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( id );
+    std::map<instanceID, uint64_t>::const_iterator bytes = _instanceSourceBytes.find( id );
+    if( !positions || positions->size() != 1 || bytes == _instanceSourceBytes.end() ||
+            bytes->second == 0 ) {
+        return std::string();
+    }
+    const instancePosition & position = positions->front();
+    if( position.section >= _dataSections.size() || !_dataSections[position.section] ) {
+        return std::string();
+    }
+    return _dataSections[position.section]->sourceRecord( position.begin, bytes->second );
+}
+
+void lazyInstMgr::observeScan( fileID file, lazyFileOffset offset, lazyFileOffset fileSize ) {
+    if( _cancelled ) return;
+    if( _cancellationCallback && _cancellationCallback() ) {
+        _cancelled = true;
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_INFO;
+        diagnostic.offset = offset;
+        diagnostic.message = "scan cancelled";
+        emitDiagnostic( diagnostic );
+        return;
+    }
+    if( _progressCallback && ( _lazyInstanceCount == 1 || ( _lazyInstanceCount % 4096 ) == 0 || offset >= fileSize ) ) {
+        LazyScanProgress progress;
+        progress.file = file;
+        progress.offset = offset;
+        progress.fileSize = fileSize;
+        progress.instancesScanned = _lazyInstanceCount;
+        _progressCallback( progress );
+    }
+}
+
+void lazyInstMgr::emitDiagnostic( LazyDiagnostic diagnostic ) {
+    std::ostringstream key;
+    key << diagnostic.type << '#' << diagnostic.entity << ':' << diagnostic.attribute << ':' << diagnostic.message;
+    uint64_t & count = _diagnosticCounts[key.str()];
+    ++count;
+    diagnostic.occurrences = count;
+    if( _diagnosticCallback && count == 1 ) {
+        _diagnosticCallback( diagnostic );
+    }
+}
+
+uint64_t lazyInstMgr::diagnosticCount( const std::string & key ) const {
+    std::map<std::string, uint64_t>::const_iterator found = _diagnosticCounts.find( key );
+    return found == _diagnosticCounts.end() ? 0 : found->second;
+}
+
+LazyCacheStatistics lazyInstMgr::cacheStatistics() const {
+    LazyCacheStatistics stats;
+    stats.instancesScanned = _lazyInstanceCount;
+    stats.instancesLoaded = _loadedInstanceCount;
+    stats.instancesPinned = _pinCounts.size();
+    stats.cacheHighWater = _cacheHighWater;
+    stats.cacheHits = _cacheHits;
+    stats.cacheMisses = _cacheMisses;
+    stats.materializations = _materializations;
+    stats.evictions = _evictions;
+    stats.activeBatches = _activeBatches;
+    stats.dataSections = _dataSections.size();
+    stats.residentSourceBytes = _residentSourceBytes;
+    stats.sourceBytesHighWater = _sourceBytesHighWater;
+    stats.cancelled = _cancelled;
+    return stats;
 }
 
 unsigned long lazyInstMgr::getNumTypes() const {
@@ -93,7 +213,7 @@ unsigned long lazyInstMgr::getNumTypes() const {
     return n ;
 }
 
-void lazyInstMgr::openFile( std::string fname ) {
+bool lazyInstMgr::openFile( std::string fname ) {
     //don't want to hold a lock for the entire time we're reading the file.
     //create a place in the vector and remember its location, then free lock
     ///FIXME begin atomic op
@@ -101,20 +221,99 @@ void lazyInstMgr::openFile( std::string fname ) {
     _files.push_back( (lazyFileReader * ) 0 );
     ///FIXME end atomic op
     lazyFileReader * lfr = new lazyFileReader( fname, this, i );
+    if( !lfr->valid() ) {
+        delete lfr;
+        _files.pop_back();
+        return false;
+    }
     _files[i] = lfr;
+    validateReferences();
     /// TODO resolve inverse attr references
     //between instances, or eDesc --> inst????
+    return true;
+}
+
+void lazyInstMgr::validateReferences() {
+    instanceRefs_t::cpair current = _fwdInstanceRefs.begin();
+    while( current.value ) {
+        instanceRefs_t::cvector::const_iterator ref = current.value->begin();
+        for( ; ref != current.value->end(); ++ref ) {
+            if( !_instanceStreamPos.find( *ref ) ) {
+                LazyDiagnostic diagnostic;
+                diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+                diagnostic.entity = current.key;
+                const char * type = typeFromFile( current.key );
+                if( type ) diagnostic.type = type;
+                instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( current.key );
+                if( positions && !positions->empty() ) diagnostic.offset = positions->front().begin;
+                std::ostringstream message;
+                message << "reference to missing instance #" << *ref;
+                diagnostic.message = message.str();
+                emitDiagnostic( diagnostic );
+            }
+        }
+        current = _fwdInstanceRefs.next();
+    }
 }
 
-SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek ) {
+SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek, bool promoteCached ) {
+    if( _batchLoadDepth && ( _cancelled ||
+            ( _cancellationCallback && _cancellationCallback() ) ) ) {
+        _cancelled = true;
+        return 0;
+    }
     assert( _mainRegistry && "Main registry has not been initialized. Do so with initRegistry() or setRegistry()." );
     std::streampos oldPos;
-    positionAndSection ps;
-    sectionID sid;
+    instancePosition pos = instancePosition();
     SDAI_Application_instance * inst = _instancesLoaded.find( id );
     if( inst ) {
+        ++_cacheHits;
+        if( promoteCached && _batchLoadDepth == 0 ) _permanentlyLoadedInstances.insert( id );
         return inst;
     }
+    ++_cacheMisses;
+    if( id > static_cast<instanceID>( INT_MAX ) ) {
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+        diagnostic.entity = id;
+        diagnostic.message = "instance ID exceeds the current SDAI materialization limit";
+        emitDiagnostic( diagnostic );
+        return 0;
+    }
+    const std::pair<std::set<instanceID>::iterator, bool> loading_result = _instancesLoading.insert( id );
+    if( !loading_result.second ) {
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+        diagnostic.entity = id;
+        const char * type = typeFromFile( id );
+        if( type ) diagnostic.type = type;
+        instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( id );
+        if( positions && !positions->empty() ) {
+            diagnostic.offset = positions->front().begin;
+        }
+        diagnostic.message = "cyclic dependency encountered while materializing instance";
+        emitDiagnostic( diagnostic );
+        return 0;
+    }
+    /* sectionReader materializes referenced instances recursively.  Keep an
+     * explicit in-progress set because an instance is not cacheable until its
+     * STEPread() has completed. */
+    class LoadingGuard {
+      public:
+        LoadingGuard( std::set<instanceID> & loading, instanceID id )
+            : _loading( loading ), _id( id ), _active( true ) {}
+        ~LoadingGuard() { release(); }
+        void release() {
+            if( _active ) {
+                _loading.erase( _id );
+                _active = false;
+            }
+        }
+      private:
+        std::set<instanceID> & _loading;
+        instanceID _id;
+        bool _active;
+    } loading_guard( _instancesLoading, id );
     instanceStreamPos_t::cvector * cv;
     if( 0 != ( cv = _instanceStreamPos.find( id ) ) ) {
         switch( cv->size() ) {
@@ -122,37 +321,186 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee
                 std::cerr << "Instance #" << id << " not found in any section." << std::endl;
                 break;
             case 1:
-                long int off;
-                ps = cv->at( 0 );
-                off = ps & 0xFFFFFFFFFFFFULL;
-                sid = ps >> 48;
-                assert( _dataSections.size() > sid );
+                pos = cv->at( 0 );
+                assert( _dataSections.size() > pos.section );
                 if( reSeek ) {
-                    oldPos = _dataSections[sid]->tellg();
+                    oldPos = _dataSections[pos.section]->tellg();
                 }
-                inst = _dataSections[sid]->getRealInstance( _mainRegistry, off, id );
+                inst = _dataSections[pos.section]->getRealInstance( _mainRegistry, pos.begin, id );
                 if( reSeek ) {
-                    _dataSections[sid]->seekg( oldPos );
+                    _dataSections[pos.section]->seekg( oldPos );
+                }
+                /* A recursive reference load may observe cancellation after
+                 * this instance began materializing.  Never cache that
+                 * partially resolved SDAI object. */
+                if( _batchLoadDepth && ( _cancelled ||
+                        ( _cancellationCallback && _cancellationCallback() ) ) ) {
+                    _cancelled = true;
+                    if( !isNilSTEPentity( inst ) ) delete inst;
+                    inst = 0;
                 }
                 break;
             default:
                 std::cerr << "Instance #" << id << " exists in multiple sections. This is not yet supported." << std::endl;
+                {
+                    LazyDiagnostic diagnostic;
+                    diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+                    diagnostic.entity = id;
+                    diagnostic.message = "instance identifier occurs in multiple DATA sections";
+                    emitDiagnostic( diagnostic );
+                }
                 break;
         }
+        /* Explicit attributes are now complete.  Inverse discovery may
+         * safely revisit this instance without tripping the materialization
+         * cycle detector. */
+        loading_guard.release();
         if( !isNilSTEPentity( inst ) ) {
             _instancesLoaded.insert( id, inst );
             _loadedInstanceCount++;
-            lazyRefs lr( this, inst );
-            lazyRefs::referentInstances_t insts = lr.result();
+            ++_materializations;
+            _cacheHighWater = std::max<uint64_t>( _cacheHighWater, _loadedInstanceCount );
+            std::map<instanceID, uint64_t>::const_iterator source_size = _instanceSourceBytes.find( id );
+            if( source_size != _instanceSourceBytes.end() ) {
+                _residentSourceBytes += source_size->second;
+                _sourceBytesHighWater = std::max<uint64_t>( _sourceBytesHighWater,
+                    _residentSourceBytes );
+            }
+            if( _batchLoadDepth ) {
+                _batchOwnedInstances.insert( id );
+            } else {
+                _permanentlyLoadedInstances.insert( id );
+                lazyRefs lr( this, inst );
+                lazyRefs::referentInstances_t insts = lr.result();
+                resolveDeferredInverses();
+            }
         } else {
             std::cerr << "Error loading instance #" << id << "." << std::endl;
+            LazyDiagnostic diagnostic;
+            diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+            diagnostic.entity = id;
+            diagnostic.offset = pos.begin;
+            diagnostic.message = "SDAI instance materialization failed";
+            emitDiagnostic( diagnostic );
         }
     } else {
         std::cerr << "Instance #" << id << " not found in any section." << std::endl;
+        LazyDiagnostic diagnostic;
+        diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+        diagnostic.entity = id;
+        diagnostic.message = "instance not found in any DATA section";
+        emitDiagnostic( diagnostic );
     }
     return inst;
 }
 
+void lazyInstMgr::resolveDeferredInverses() {
+    if( !_instancesLoading.empty() ) return;
+
+    while( !_deferredInverseInstances.empty() ) {
+        std::set<instanceID> deferred;
+        deferred.swap( _deferredInverseInstances );
+        std::set<instanceID>::const_iterator id = deferred.begin();
+        for( ; id != deferred.end(); ++id ) {
+            SDAI_Application_instance * inst = cachedInstance( *id );
+            if( inst ) {
+                lazyRefs refs( this, inst );
+            }
+        }
+    }
+}
+
+SDAI_Application_instance * lazyInstMgr::cachedInstance( instanceID id ) {
+    return _instancesLoaded.find( id );
+}
+
+std::vector<instanceID> lazyInstMgr::dependencyClosure( const std::vector<instanceID> & roots ) {
+    std::set<instanceID> closure( roots.begin(), roots.end() );
+    std::vector<instanceID> queue( roots.begin(), roots.end() );
+    size_t current = 0;
+    while( current < queue.size() ) {
+        if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) {
+            _cancelled = true;
+            break;
+        }
+        instanceRefs_t::cvector * refs = _fwdInstanceRefs.find( queue[current++] );
+        if( !refs ) continue;
+        instanceRefs_t::cvector::const_iterator ref = refs->begin();
+        for( ; ref != refs->end(); ++ref ) {
+            if( closure.insert( *ref ).second ) queue.push_back( *ref );
+        }
+    }
+    return std::vector<instanceID>( closure.begin(), closure.end() );
+}
+
+LazyInstanceBatch lazyInstMgr::loadBatch( instanceID root ) {
+    std::vector<instanceID> roots( 1, root );
+    return loadBatch( roots );
+}
+
+LazyInstanceBatch lazyInstMgr::loadBatch( const std::vector<instanceID> & roots ) {
+    std::vector<instanceID> closure = dependencyClosure( roots );
+    std::vector<instanceID>::const_iterator id = closure.begin();
+    for( ; id != closure.end(); ++id ) {
+        if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) {
+            _cancelled = true;
+            closure.erase( id, closure.end() );
+            break;
+        }
+        ++_pinCounts[*id];
+    }
+    ++_activeBatches;
+    ++_batchLoadDepth;
+    for( id = closure.begin(); id != closure.end(); ++id ) {
+        if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) {
+            _cancelled = true;
+            break;
+        }
+        if( !loadInstance( *id, true ) ) {
+            LazyDiagnostic diagnostic;
+            diagnostic.severity = LAZY_DIAGNOSTIC_ERROR;
+            diagnostic.entity = *id;
+            diagnostic.message = "dependency batch could not materialize instance";
+            emitDiagnostic( diagnostic );
+        }
+    }
+    --_batchLoadDepth;
+    return LazyInstanceBatch( this, roots, closure );
+}
+
+void lazyInstMgr::releaseBatch( const std::vector<instanceID> & instances ) {
+    std::vector<instanceID> evict;
+    std::vector<instanceID>::const_iterator id = instances.begin();
+    for( ; id != instances.end(); ++id ) {
+        std::map<instanceID, size_t>::iterator pin = _pinCounts.find( *id );
+        if( pin == _pinCounts.end() ) continue;
+        if( --pin->second == 0 ) {
+            _pinCounts.erase( pin );
+            if( _batchOwnedInstances.count( *id ) && !_permanentlyLoadedInstances.count( *id ) ) {
+                evict.push_back( *id );
+            }
+        }
+    }
+    std::vector<SDAI_Application_instance *> deleted;
+    for( id = evict.begin(); id != evict.end(); ++id ) {
+        SDAI_Application_instance * inst = _instancesLoaded.find( *id );
+        if( !inst ) continue;
+        _instancesLoaded.removeEntry( *id );
+        _batchOwnedInstances.erase( *id );
+        deleted.push_back( inst );
+        --_loadedInstanceCount;
+        std::map<instanceID, uint64_t>::const_iterator source_size = _instanceSourceBytes.find( *id );
+        if( source_size != _instanceSourceBytes.end() ) {
+            _residentSourceBytes = source_size->second <= _residentSourceBytes ?
+                _residentSourceBytes - source_size->second : 0;
+        }
+        ++_evictions;
+    }
+    std::vector<SDAI_Application_instance *>::iterator inst = deleted.begin();
+    for( ; inst != deleted.end(); ++inst ) delete *inst;
+    if( _activeBatches ) --_activeBatches;
+}
+
 
 instanceSet * lazyInstMgr::instanceDependencies( instanceID id ) {
     instanceSet * checkedDependencies = new instanceSet();
@@ -182,4 +530,3 @@ instanceSet * lazyInstMgr::instanceDependencies( instanceID id ) {
 
     return checkedDependencies;
 }
-
diff --git a/src/cllazyfile/lazyP21DataSectionReader.cc b/src/cllazyfile/lazyP21DataSectionReader.cc
index 0716f5c5f..a2c201923 100644
--- a/src/cllazyfile/lazyP21DataSectionReader.cc
+++ b/src/cllazyfile/lazyP21DataSectionReader.cc
@@ -3,13 +3,60 @@
 #include "cllazyfile/lazyP21DataSectionReader.h"
 #include "cllazyfile/lazyInstMgr.h"
 
+void lazyP21DataSectionReader::findSectionStart() {
+    std::streampos keywordEnd = findNormalString( "DATA" );
+    if( keywordEnd == std::streampos( -1 ) ) {
+        _sectionStart = -1;
+        return;
+    }
+    _file.seekg( keywordEnd );
+    skipWS();
+    int depth = 0;
+    int current = _file.get();
+    if( current == ';' ) {
+        _sectionStart = _file.tellg();
+        return;
+    }
+    if( current != '(' ) {
+        _sectionStart = -1;
+        return;
+    }
+    depth = 1;
+    while( depth && _file.good() ) {
+        current = _file.get();
+        if( current == '\'' ) {
+            _file.seekg( _file.tellg() - std::streampos( 1 ) );
+            GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() );
+        } else if( current == '/' && _file.peek() == '*' ) {
+            findNormalString( "*/" );
+        } else if( current == '(' ) {
+            ++depth;
+        } else if( current == ')' ) {
+            --depth;
+        }
+    }
+    skipWS();
+    if( depth == 0 && _file.get() == ';' ) {
+        _sectionStart = _file.tellg();
+    } else {
+        _sectionStart = -1;
+    }
+}
+
 lazyP21DataSectionReader::lazyP21DataSectionReader( lazyFileReader * parent, std::ifstream & file,
         std::streampos start, sectionID sid ):
     lazyDataSectionReader( parent, file, start, sid ) {
     findSectionStart();
+    if( _sectionStart == std::streampos( -1 ) ) {
+        _error = true;
+        return;
+    }
     namedLazyInstance nl;
     while( nl = nextInstance(), ( ( nl.loc.begin > 0 ) && ( nl.name != 0 ) ) ) {
         parent->getInstMgr()->addLazyInstance( nl );
+        parent->getInstMgr()->observeScan( parent->ID(),
+            static_cast<lazyFileOffset>( static_cast<std::streamoff>( _file.tellg() ) ), parent->fileSize() );
+        if( parent->getInstMgr()->cancelled() ) return;
     }
 
     if(  sectionReader::_error->severity() <= SEVERITY_WARNING ) {
@@ -49,26 +96,64 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() {
     namedLazyInstance i;
 
     i.refs = 0;
+    i.componentTypes = 0;
     i.loc.section = 0;
-    i.loc.begin = _file.tellg();
+    i.loc.end = 0;
+    std::streampos start = _file.tellg();
+    i.loc.begin = start == std::streampos( -1 ) ? 0 :
+        static_cast<lazyFileOffset>( static_cast<std::streamoff>( start ) );
     i.loc.instance = readInstanceNumber();
     if( ( _file.good() ) && ( i.loc.instance > 0 ) ) {
-        skipWS();
+        if( !skipTokenSeparators() ) {
+            _file.setstate( std::ios::failbit );
+        }
+        if( _file.good() && _file.peek() == '&' && !indexScope() ) {
+            _file.setstate( std::ios::failbit );
+        }
+        if( _file.good() && !skipTokenSeparators() ) {
+            _file.setstate( std::ios::failbit );
+        }
         i.loc.section = _sectionID;
-        i.name = getDelimitedKeyword( ";( /\\" );
+        if( _file.good() ) i.name = getDelimitedKeyword( ";( /\\" );
         if( _file.good() ) {
-            end = seekInstanceEnd( & i.refs );
+            if( i.name[0] == '\0' ) i.componentTypes = new std::vector<std::string>;
+            end = seekInstanceEnd( & i.refs, i.componentTypes );
+            if( end != std::streampos( -1 ) ) {
+                i.loc.end = static_cast<lazyFileOffset>( static_cast<std::streamoff>( end ) );
+            }
         }
     }
     if( ( i.loc.instance == 0 ) || ( !_file.good() ) || ( end == ( std::streampos ) - 1 ) ) {
         //invalid instance, so clear everything
         _file.seekg( i.loc.begin );
-        i.loc.begin = -1;
+        i.loc.begin = 0;
         if( i.refs ) {
             delete i.refs;
         }
+        if( i.componentTypes ) {
+            delete i.componentTypes;
+            i.componentTypes = 0;
+        }
         i.name = 0;
     }
     return i;
 }
+bool lazyP21DataSectionReader::indexScope() {
+    if( !skipTokenSeparators() || _file.get() != '&' ) return false;
+    const std::string scopeKeyword = getDelimitedKeyword( "#/(;\\" );
+    if( scopeKeyword != "SCOPE" ) return false;
+
+    bool haveInstance = false;
+    while( skipTokenSeparators() && _file.peek() == '#' ) {
+        namedLazyInstance nested = nextInstance();
+        if( nested.loc.begin == 0 || !nested.name ) return false;
+        _lazyFile->getInstMgr()->addLazyInstance( nested );
+        haveInstance = true;
+    }
+    if( !haveInstance || !skipTokenSeparators() ) return false;
+
+    const std::string endKeyword = getDelimitedKeyword( "/#(;\\" );
+    if( endKeyword != "ENDSCOPE" ) return false;
+    return skipScopeExportList();
+}
 
diff --git a/src/cllazyfile/lazyRefs.h b/src/cllazyfile/lazyRefs.h
index b8daff295..d005bc01b 100644
--- a/src/cllazyfile/lazyRefs.h
+++ b/src/cllazyfile/lazyRefs.h
@@ -87,6 +87,16 @@ class SC_LAZYFILE_EXPORT lazyRefs {
             }
             //3c - for each item in both _refMap and edL, add it to _referentInstances
             potentialReferentInsts( edL );
+            /* Do not partially populate an inverse attribute and then repeat
+             * the whole pass.  A referrer that is still being parsed cannot
+             * be inspected reliably, so defer this instance as a unit. */
+            referentInstances_t::const_iterator pending = _referentInstances.begin();
+            for( ; pending != _referentInstances.end(); ++pending ) {
+                if( _lim->isMaterializing( *pending ) ) {
+                    _lim->deferInverseResolution( _id );
+                    return;
+                }
+            }
             //3d - load each inst
             iAstruct ias = invAttr( _inst, ia );
             referentInstances_t::iterator insts = _referentInstances.begin();
@@ -98,7 +108,12 @@ class SC_LAZYFILE_EXPORT lazyRefs {
 
         void loadInstIFFreferent( instanceID inst, iAstruct ias, const Inverse_attribute * ia ) {
             bool prevLoaded = _lim->isLoaded( inst );
+            if( _lim->isMaterializing( inst ) ) {
+                _lim->deferInverseResolution( _id );
+                return;
+            }
             SDAI_Application_instance * rinst = _lim->loadInstance( inst );
+            if( !rinst ) return;
             bool ref = refersToCurrentInst( ia, rinst );
             if( ref ) {
                 if( ia->inverted_attr_()->IsAggrType() ) {
@@ -108,8 +123,16 @@ class SC_LAZYFILE_EXPORT lazyRefs {
                         assert( invAttr( _inst, ia ).a == ias.a );
                     }
                     EntityAggregate * ea = ias.a;
-                    //TODO check if duplicate
-                    ea->AddNode( new EntityNode( rinst ) );
+                    bool duplicate = false;
+                    EntityNode * existing = static_cast<EntityNode *>( ea->GetHead() );
+                    while( existing ) {
+                        if( existing->node == rinst ) {
+                            duplicate = true;
+                            break;
+                        }
+                        existing = static_cast<EntityNode *>( existing->NextNode() );
+                    }
+                    if( !duplicate ) ea->AddNode( new EntityNode( rinst ) );
                 } else {
                     SDAI_Application_instance * ai = ias.i;
                     if( !ai ) {
@@ -131,8 +154,10 @@ class SC_LAZYFILE_EXPORT lazyRefs {
 
         ///3e - check if actually inverse ref
         bool refersToCurrentInst( const Inverse_attribute * ia, SDAI_Application_instance * referrer ) {
+            if( !referrer ) return false;
             //find the attr
             int rindex = attrIndex( referrer, ia->_inverted_attr_id, ia->_inverted_entity_id );
+            if( rindex < 0 ) return false;
             STEPattribute sa = referrer->attributes[ rindex ];
             assert( sa.getADesc()->BaseType() == ENTITY_TYPE );
             bool found = false;
@@ -279,7 +304,7 @@ class SC_LAZYFILE_EXPORT lazyRefs {
 
 
             // 1. find inverse attrs with recursion
-            getInverseAttrs( ai->eDesc, _iaList );
+            getInverseAttrs( _inst->eDesc, _iaList );
 
             //2. find reverse refs, map id to type (stop if there are no inverse attrs or no refs)
             if( _iaList.size() == 0 || !mapRefsToTypes() ) {
diff --git a/src/cllazyfile/lazySupport.cc b/src/cllazyfile/lazySupport.cc
new file mode 100644
index 000000000..ebebe7b0e
--- /dev/null
+++ b/src/cllazyfile/lazySupport.cc
@@ -0,0 +1,106 @@
+#include <stdexcept>
+#include <utility>
+#include <ostream>
+
+#include "cllazyfile/lazySupport.h"
+#include "cllazyfile/lazyInstMgr.h"
+
+namespace {
+const instanceRefs & emptyIds() {
+    static const instanceRefs ids;
+    return ids;
+}
+}
+
+LazyInstanceIdView::const_iterator LazyInstanceIdView::begin() const {
+    return _ids ? _ids->begin() : emptyIds().begin();
+}
+
+LazyInstanceIdView::const_iterator LazyInstanceIdView::end() const {
+    return _ids ? _ids->end() : emptyIds().end();
+}
+
+size_t LazyInstanceIdView::size() const {
+    return _ids ? _ids->size() : 0;
+}
+
+bool LazyInstanceIdView::empty() const {
+    return size() == 0;
+}
+
+instanceID LazyInstanceIdView::operator[]( size_t index ) const {
+    if( !_ids || index >= _ids->size() ) {
+        throw std::out_of_range( "LazyInstanceIdView index" );
+    }
+    return ( *_ids )[index];
+}
+
+LazyInstanceBatch::LazyInstanceBatch(): _manager( 0 ) {}
+
+LazyInstanceBatch::LazyInstanceBatch( lazyInstMgr * manager,
+        const std::vector<instanceID> & roots, const std::vector<instanceID> & instances ):
+    _manager( manager ), _roots( roots ), _instances( instances ) {}
+
+LazyInstanceBatch::~LazyInstanceBatch() {
+    release();
+}
+
+LazyInstanceBatch::LazyInstanceBatch( LazyInstanceBatch && other ):
+    _manager( other._manager ), _roots( std::move( other._roots ) ),
+    _instances( std::move( other._instances ) ) {
+    other._manager = 0;
+}
+
+LazyInstanceBatch & LazyInstanceBatch::operator=( LazyInstanceBatch && other ) {
+    if( this != &other ) {
+        release();
+        _manager = other._manager;
+        _roots = std::move( other._roots );
+        _instances = std::move( other._instances );
+        other._manager = 0;
+    }
+    return *this;
+}
+
+bool LazyInstanceBatch::valid() const {
+    return _manager != 0;
+}
+
+void LazyInstanceBatch::release() {
+    if( _manager ) {
+        _manager->releaseBatch( _instances );
+        _manager = 0;
+    }
+    _roots.clear();
+    _instances.clear();
+}
+
+const std::vector<instanceID> & LazyInstanceBatch::roots() const {
+    return _roots;
+}
+
+const std::vector<instanceID> & LazyInstanceBatch::instances() const {
+    return _instances;
+}
+
+SDAI_Application_instance * LazyInstanceBatch::get( instanceID id ) const {
+    return _manager ? _manager->cachedInstance( id ) : 0;
+}
+
+LazyTextDiagnosticAdapter::LazyTextDiagnosticAdapter( std::ostream & stream ): _stream( &stream ) {}
+
+void LazyTextDiagnosticAdapter::operator()( const LazyDiagnostic & diagnostic ) const {
+    const char * label = "info";
+    if( diagnostic.severity == LAZY_DIAGNOSTIC_WARNING ) label = "warning";
+    if( diagnostic.severity == LAZY_DIAGNOSTIC_ERROR ) label = "error";
+    if( diagnostic.severity == LAZY_DIAGNOSTIC_FATAL ) label = "fatal";
+    ( *_stream ) << label;
+    if( diagnostic.entity ) ( *_stream ) << " #" << diagnostic.entity;
+    if( !diagnostic.type.empty() ) ( *_stream ) << " " << diagnostic.type;
+    if( diagnostic.offset ) ( *_stream ) << " at offset " << diagnostic.offset;
+    if( diagnostic.line ) ( *_stream ) << " line " << diagnostic.line;
+    if( !diagnostic.attribute.empty() ) ( *_stream ) << " attribute " << diagnostic.attribute;
+    ( *_stream ) << ": " << diagnostic.message;
+    if( diagnostic.occurrences > 1 ) ( *_stream ) << " (" << diagnostic.occurrences << " occurrences)";
+    ( *_stream ) << std::endl;
+}
diff --git a/src/cllazyfile/lazy_index_test.cc b/src/cllazyfile/lazy_index_test.cc
new file mode 100644
index 000000000..cb804963e
--- /dev/null
+++ b/src/cllazyfile/lazy_index_test.cc
@@ -0,0 +1,99 @@
+#include <cstdlib>
+#include <iostream>
+
+#include "cllazyfile/lazyInstMgr.h"
+
+namespace {
+void require( bool condition, const char * message ) {
+    if( !condition ) {
+        std::cerr << "lazy_index_test: " << message << std::endl;
+        std::exit( EXIT_FAILURE );
+    }
+}
+}
+
+int main( int argc, char ** argv ) {
+    require( argc == 3, "expected ordinary and scoped fixture paths" );
+
+    lazyInstMgr manager;
+    uint64_t progressCalls = 0;
+    uint64_t diagnosticCalls = 0;
+    LazyDiagnostic missingReference;
+    manager.setProgressCallback( [&progressCalls]( const LazyScanProgress & progress ) {
+        require( progress.fileSize >= progress.offset, "progress offset exceeds file size" );
+        ++progressCalls;
+    } );
+    manager.setDiagnosticCallback( [&diagnosticCalls, &missingReference]( const LazyDiagnostic & diagnostic ) {
+        ++diagnosticCalls;
+        if( diagnostic.entity == 5 ) missingReference = diagnostic;
+    } );
+    manager.openFile( argv[1] );
+
+    LazyCacheStatistics stats = manager.cacheStatistics();
+    require( stats.instancesScanned == 5, "wrong scan count" );
+    require( stats.dataSections == 2, "second DATA section was not indexed" );
+    require( progressCalls > 0, "progress callback was not called" );
+    require( manager.instancesByType( "a" ).size() == 1, "case-insensitive type index failed" );
+    require( manager.allInstances().size() == manager.totalInstanceCount(), "all-instance index count failed" );
+    require( manager.instancesByType( "C" ).size() == 1, "first complex component not indexed" );
+    require( manager.instancesByType( "D" ).size() == 1, "second complex component not indexed" );
+    require( manager.instancesByType( "" ).size() == 1, "complex instance index failed" );
+    require( diagnosticCalls == 1, "structured diagnostic callback was not bounded" );
+    require( missingReference.severity == LAZY_DIAGNOSTIC_ERROR && missingReference.offset > 0,
+        "missing-reference diagnostic lacks structured context" );
+    require( missingReference.message == "reference to missing instance #99", "wrong missing-reference diagnostic" );
+    require( manager.forwardReferences( 1 ).size() == 1 && manager.forwardReferences( 1 )[0] == 2,
+        "forward reference index failed" );
+    require( manager.reverseReferences( 1 ).size() == 1 && manager.reverseReferences( 1 )[0] == 3,
+        "reverse reference index failed" );
+    const std::string firstSource = manager.sourceRecord( 1 );
+    const std::string secondSource = manager.sourceRecord( 2 );
+    require( firstSource.find( "#1=A(" ) != std::string::npos &&
+        !firstSource.empty() && firstSource[firstSource.size() - 1] == ';',
+        "first exact source record failed" );
+    require( secondSource.find( "#2=B();" ) != std::string::npos &&
+        !secondSource.empty() && secondSource[secondSource.size() - 1] == ';',
+        "second exact source record or stream restoration failed" );
+    require( manager.sourceRecord( 99 ).empty(),
+        "missing source record was not empty" );
+
+    lazyInstMgr cancelled;
+    uint64_t cancellationCalls = 0;
+    cancelled.setCancellationCallback( [&cancellationCalls]() {
+        return ++cancellationCalls >= 2;
+    } );
+    cancelled.openFile( argv[1] );
+    stats = cancelled.cacheStatistics();
+    require( stats.cancelled, "cancellation was not recorded" );
+    require( stats.instancesScanned == 2, "scan did not stop at cancellation boundary" );
+
+    lazyInstMgr scoped;
+    require( scoped.openFile( argv[2] ), "scoped fixture did not open" );
+    stats = scoped.cacheStatistics();
+    require( stats.instancesScanned == 5, "wrong scoped scan count" );
+    require( scoped.instancesByType( "A" ).size() == 3,
+        "scoped simple instances were not indexed" );
+    require( scoped.instancesByType( "B" ).size() == 1,
+        "nested scoped instance was not indexed" );
+    require( scoped.instancesByType( "C" ).size() == 1 &&
+        scoped.instancesByType( "D" ).size() == 1 &&
+        scoped.instancesByType( "" ).size() == 1,
+        "scoped complex owner was not indexed" );
+    require( scoped.forwardReferences( 11 ).size() == 1 &&
+        scoped.forwardReferences( 11 )[0] == 12,
+        "nested scoped reference was not indexed" );
+    require( scoped.forwardReferences( 20 ).size() == 1 &&
+        scoped.forwardReferences( 20 )[0] == 21,
+        "complex scope-owner reference was not indexed" );
+    const std::string nestedOwner = scoped.sourceRecord( 11 );
+    const std::string complexOwner = scoped.sourceRecord( 20 );
+    require( nestedOwner.find( "#11=&SCOPE" ) != std::string::npos &&
+        nestedOwner.find( "ENDSCOPE /#12/ A(" ) != std::string::npos &&
+        !nestedOwner.empty() && nestedOwner[nestedOwner.size() - 1] == ';',
+        "nested scope source record was not preserved" );
+    require( complexOwner.find( "#20=&SCOPE" ) != std::string::npos &&
+        complexOwner.find( "ENDSCOPE (C()D(" ) != std::string::npos &&
+        !complexOwner.empty() && complexOwner[complexOwner.size() - 1] == ';',
+        "complex scope source record was not preserved" );
+    return EXIT_SUCCESS;
+}
diff --git a/src/cllazyfile/lazy_test.cc b/src/cllazyfile/lazy_test.cc
index d5ce6081a..246f7429f 100644
--- a/src/cllazyfile/lazy_test.cc
+++ b/src/cllazyfile/lazy_test.cc
@@ -171,6 +171,34 @@ int main( int argc, char ** argv ) {
 #ifndef NO_REGISTRY
     if( instWithRef ) {
         std::cout << "Number of data section instances fully loaded: " << mgr->loadedInstanceCount() << std::endl;
+        {
+            LazyInstanceBatch first = mgr->loadBatch( instWithRef );
+            LazyInstanceBatch second = mgr->loadBatch( instWithRef );
+            size_t sharedSize = second.instances().size();
+            first.release();
+            if( mgr->loadedInstanceCount() != sharedSize || !second.get( instWithRef ) ) {
+                std::cerr << "Releasing one batch evicted instances pinned by another batch" << std::endl;
+                return EXIT_FAILURE;
+            }
+        }
+        if( mgr->loadedInstanceCount() != 0 ) {
+            std::cerr << "Shared dependency batches did not release their cache" << std::endl;
+            return EXIT_FAILURE;
+        }
+        for( int cycle = 0; cycle < 3; ++cycle ) {
+            {
+                LazyInstanceBatch batch = mgr->loadBatch( instWithRef );
+                if( !batch.valid() || !batch.get( instWithRef ) ) {
+                    std::cerr << "Unable to materialize dependency batch for #" << instWithRef << std::endl;
+                    return EXIT_FAILURE;
+                }
+                std::cout << "Batch-pinned instances: " << batch.instances().size() << std::endl;
+            }
+            if( mgr->loadedInstanceCount() != 0 ) {
+                std::cerr << "Dependency batch did not return the cache to its initial size" << std::endl;
+                return EXIT_FAILURE;
+            }
+        }
         std::cout << "Loading #" << instWithRef;
         SDAI_Application_instance * inst = mgr->loadInstance( instWithRef );
         std::cout << " which is of type " << inst->EntityName() << std::endl;
@@ -193,4 +221,3 @@ int main( int argc, char ** argv ) {
     delete mgr;
     //stats will print from its destructor
 }
-
diff --git a/src/cllazyfile/p21HeaderSectionReader.cc b/src/cllazyfile/p21HeaderSectionReader.cc
index 037604fc4..e8fa784d3 100644
--- a/src/cllazyfile/p21HeaderSectionReader.cc
+++ b/src/cllazyfile/p21HeaderSectionReader.cc
@@ -34,7 +34,11 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() {
     static instanceID nextFreeInstance = 4; // 1-3 are reserved per 10303-21
 
     i.refs = 0;
-    i.loc.begin = _file.tellg();
+    i.componentTypes = 0;
+    i.loc.end = 0;
+    std::streampos start = _file.tellg();
+    i.loc.begin = start == std::streampos( -1 ) ? 0 :
+        static_cast<lazyFileOffset>( static_cast<std::streamoff>( start ) );
     i.loc.section = _sectionID;
     skipWS();
     if( i.loc.begin <= 0 ) {
@@ -55,12 +59,14 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() {
         assert( strlen( i.name ) > 0 );
 
         std::streampos end = seekInstanceEnd( 0 ); //no references in file header
+        if( end != std::streampos( -1 ) ) {
+            i.loc.end = static_cast<lazyFileOffset>( static_cast<std::streamoff>( end ) );
+        }
         if( ( (signed long int)end == -1 ) || ( end >= _sectionEnd ) ) {
             //invalid instance, so clear everything
-            i.loc.begin = -1;
+            i.loc.begin = 0;
             i.name = 0;
         }
     }
     return i;
 }
-
diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc
index 4a159439a..e22014598 100644
--- a/src/cllazyfile/sectionReader.cc
+++ b/src/cllazyfile/sectionReader.cc
@@ -10,6 +10,7 @@
 #include <string>
 #include <assert.h>
 #include <limits.h>
+#include <limits>
 
 #ifdef _WIN32
 #  define strtoull _strtoui64
@@ -39,6 +40,112 @@ sectionReader::~sectionReader() {
      delete _error;
 }
 
+bool sectionReader::skipComment() {
+    int previous = 0;
+    int current;
+    while( current = _file.get(), _file.good() ) {
+        if( previous == '*' && current == '/' ) {
+            return true;
+        }
+        previous = current;
+    }
+    return false;
+}
+
+
+bool sectionReader::skipTokenSeparators() {
+    while( _file.good() ) {
+        skipWS();
+        if( _file.peek() != '/' ) return _file.good();
+        const std::streampos slash = _file.tellg();
+        _file.get();
+        if( _file.peek() != '*' ) {
+            _file.seekg( slash );
+            return _file.good();
+        }
+        _file.get(); // consume the opening star
+        if( !skipComment() ) return false;
+    }
+    return false;
+}
+
+
+bool sectionReader::skipScopeExportList() {
+    if( !skipTokenSeparators() ) return false;
+    if( _file.peek() != '/' ) return true;
+    _file.get();
+
+    bool haveExport = false;
+    while( _file.good() ) {
+        if( !skipTokenSeparators() || _file.get() != '#' ) return false;
+        bool haveDigit = false;
+        bool haveNonzeroDigit = false;
+        while( _file.good() && isdigit( _file.peek() ) ) {
+            const int digit = _file.get();
+            haveDigit = true;
+            if( digit != '0' ) haveNonzeroDigit = true;
+        }
+        if( !haveDigit || !haveNonzeroDigit || !skipTokenSeparators() ) return false;
+        haveExport = true;
+        const int delimiter = _file.get();
+        if( delimiter == '/' ) return haveExport;
+        if( delimiter != ',' ) return false;
+    }
+    return false;
+}
+
+
+bool sectionReader::skipScope() {
+    if( !skipTokenSeparators() || _file.get() != '&' ) return false;
+    skipWS();
+    static const char scopeKeyword[] = "SCOPE";
+    for( size_t i = 0; scopeKeyword[i]; ++i ) {
+        if( _file.get() != scopeKeyword[i] ) return false;
+    }
+
+    int depth = 1;
+    while( depth > 0 && _file.good() ) {
+        const int current = _file.get();
+        if( current == '\'' ) {
+            _file.seekg( _file.tellg() - std::streampos( 1 ) );
+            GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() );
+            continue;
+        }
+        if( current == '/' && _file.peek() == '*' ) {
+            _file.get();
+            if( !skipComment() ) return false;
+            continue;
+        }
+        if( current == '&' ) {
+            const std::streampos afterAmpersand = _file.tellg();
+            skipWS();
+            std::string keyword;
+            while( _file.good() && ( isupper( _file.peek() ) ||
+                    isdigit( _file.peek() ) || _file.peek() == '_' ||
+                    _file.peek() == '-' ) ) {
+                keyword.push_back( static_cast<char>( _file.get() ) );
+            }
+            if( keyword == "SCOPE" ) {
+                ++depth;
+            } else {
+                _file.seekg( afterAmpersand );
+            }
+            continue;
+        }
+        if( isupper( current ) ) {
+            std::string keyword( 1, static_cast<char>( current ) );
+            while( _file.good() && ( isupper( _file.peek() ) ||
+                    isdigit( _file.peek() ) || _file.peek() == '_' ||
+                    _file.peek() == '-' ) ) {
+                keyword.push_back( static_cast<char>( _file.get() ) );
+            }
+            if( keyword == "ENDSCOPE" ) --depth;
+        }
+    }
+    return depth == 0 && skipScopeExportList();
+}
+
+
 std::streampos sectionReader::findNormalString( const std::string & str, bool semicolon ) {
     std::streampos found = -1, startPos = _file.tellg(), nextTry = startPos;
     int i = 0, l = str.length();
@@ -63,8 +170,11 @@ std::streampos sectionReader::findNormalString( const std::string & str, bool se
             GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() );
         }
         if( ( c == '/' ) && ( _file.peek() == '*' ) ) {
-            //push past comment
-            findNormalString( "*/" );
+            _file.get(); // consume the opening star
+            if( !skipComment() ) {
+                return -1;
+            }
+            continue;
         }
         if( str[i] == c ) {
             i++;
@@ -92,6 +202,32 @@ std::streampos sectionReader::findNormalString( const std::string & str, bool se
 }
 
 
+std::string sectionReader::sourceRecord( lazyFileOffset begin, uint64_t length ) {
+    if( begin == 0 || length == 0 ||
+            length > static_cast<uint64_t>( std::numeric_limits<std::streamsize>::max() ) ||
+            length > static_cast<uint64_t>( std::numeric_limits<size_t>::max() ) ) {
+        return std::string();
+    }
+
+    const std::streampos saved = _file.tellg();
+    _file.clear();
+    _file.seekg( static_cast<std::streamoff>( begin ) );
+    if( !_file.good() ) {
+        _file.clear();
+        if( saved != std::streampos( -1 ) ) _file.seekg( saved );
+        return std::string();
+    }
+
+    std::string source( static_cast<size_t>( length ), '\0' );
+    _file.read( &source[0], static_cast<std::streamsize>( length ) );
+    const bool complete = static_cast<uint64_t>( _file.gcount() ) == length;
+
+    _file.clear();
+    if( saved != std::streampos( -1 ) ) _file.seekg( saved );
+    return complete ? source : std::string();
+}
+
+
 //NOTE different behavior than const char * GetKeyword( istream & in, const char * delims, ErrorDescriptor & err ) in read_func.cc
 // returns pointer to the contents of a static std::string
 const char * sectionReader::getDelimitedKeyword( const char * delimiters ) {
@@ -106,7 +242,10 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) {
             str.append( 1, c );
         } else if( ( c == '/' ) && ( _file.peek() == '*' ) && ( str.length() == 0 ) ) {
             //push past comment
-            findNormalString( "*/" );
+            _file.get(); // consume the opening star
+            if( !skipComment() ) {
+                break;
+            }
             skipWS();
             continue;
         } else {
@@ -115,7 +254,7 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) {
         }
     }
     c = _file.peek();
-    if( !strchr( delimiters, c ) ) {
+    if( !strchr( delimiters, c ) && !isspace( static_cast<unsigned char>( c ) ) ) {
         std::cerr << SC_CURRENT_FUNCTION << ": missing delimiter. Found " << c << ", expected one of " << delimiters << " at end of keyword " << str << ". File offset: " << _file.tellg() << std::endl;
         abort();
     }
@@ -125,17 +264,22 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) {
 /// search forward in the file for the end of the instance. Start position should
 /// be the opening parenthesis; otherwise, it is likely to fail.
 ///NOTE *must* check return value!
-std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) {
+std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs, std::vector<std::string> * componentTypes ) {
     int c;
     int parenDepth = 0;
+    bool expectComplexType = false;
     while( c = _file.get(), _file.good() ) {
         switch( c ) {
             case '(':
                 parenDepth++;
+                if( componentTypes && parenDepth == 1 ) expectComplexType = true;
                 break;
             case '/':
                 if( _file.peek() == '*' ) {
-                    findNormalString( "*/" );
+                    _file.get(); // consume the opening star
+                    if( !skipComment() ) {
+                        return -1;
+                    }
                 } else {
                     return -1;
                 }
@@ -162,7 +306,8 @@ std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) {
                 }
                 break;
             case ')':
-                if( --parenDepth == 0 ) {
+                if( --parenDepth == 1 && componentTypes ) expectComplexType = true;
+                if( parenDepth == 0 ) {
                     skipWS();
                     if( _file.get() == ';' ) {
                         return _file.tellg();
@@ -170,7 +315,23 @@ std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) {
                         _file.seekg( _file.tellg() - std::streampos(1) );
                     }
                 }
+                break;
             default:
+                if( componentTypes && parenDepth == 1 && expectComplexType &&
+                        ( isupper( c ) || c == '!' ) ) {
+                    std::string type( 1, static_cast<char>( c ) );
+                    while( _file.good() ) {
+                        int next = _file.get();
+                        if( next == '-' || next == '_' || isupper( next ) || isdigit( next ) ) {
+                            type.push_back( static_cast<char>( next ) );
+                        } else {
+                            _file.putback( static_cast<char>( next ) );
+                            break;
+                        }
+                    }
+                    componentTypes->push_back( type );
+                    expectComplexType = false;
+                }
                 break;
         }
     }
@@ -196,7 +357,10 @@ instanceID sectionReader::readInstanceNumber() {
     skipWS();
     c = _file.get();
     if( ( c == '/' ) && ( _file.peek() == '*' ) ) {
-        findNormalString( "*/" );
+        _file.get(); // consume the opening star
+        if( !skipComment() ) {
+            return 0;
+        }
     } else {
         _file.seekg( _file.tellg() - std::streampos(1) );
     }
@@ -261,10 +425,11 @@ instanceID sectionReader::readInstanceNumber() {
 /** load an instance and return a pointer to it.
  * side effect: recursively loads any instances the specified instance depends upon
  */
-SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg, long int begin, instanceID instance,
+SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg, lazyFileOffset begin, instanceID instance,
         const std::string & typeName, const std::string & schName, bool header ) {
     int c;
     const char * tName = 0, * sName = 0; //these are necessary since typeName and schName are const
+    std::string normalizedSchema;
     std::string comment;
     Severity sev = SEVERITY_NULL;
     SDAI_Application_instance * inst = 0;
@@ -277,7 +442,10 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg
         if( fs ) {
             StringNode * sn = ( StringNode * ) fs->schema_identifiers_()->GetHead();
             if( sn ) {
-                sName = sn->value.c_str();
+                normalizedSchema = sn->value.c_str();
+                size_t qualifier = normalizedSchema.find_first_of( " {" );
+                if( qualifier != std::string::npos ) normalizedSchema.erase( qualifier );
+                sName = normalizedSchema.c_str();
                 if( sn->NextNode() ) {
                     std::cerr << "Warning - multiple schema names found. Only searching with first one." << std::endl;
                 }
@@ -287,7 +455,7 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg
         }
     }
 
-    _file.seekg( begin );
+    _file.seekg( static_cast<std::streamoff>( begin ) );
     skipWS();
     ReadTokenSeparator( _file, &comment );
     if( !header ) {
@@ -296,11 +464,11 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg
     skipWS();
     ReadTokenSeparator( _file, &comment );
     c = _file.peek();
+    if( c == '&' ) {
+        if( !skipScope() || !skipTokenSeparators() ) return 0;
+        c = _file.peek();
+    }
     switch( c ) {
-        case '&':
-            std::cerr << "Can't handle scope instances. Skipping #" << instance << ", offset " << _file.tellg() << std::endl;
-            // sev = CreateScopeInstances( in, &scopelist );
-            break;
         case '(':
             inst = CreateSubSuperInstance( reg, instance, sev );
             break;
@@ -311,6 +479,11 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg
             if( ( !header ) && ( typeName.size() == 0 ) ) {
                 tName = getDelimitedKeyword( ";( /\\" );
             }
+            std::string materializationType;
+            if( !header && tName ) {
+                materializationType = _lazyFile->getInstMgr()->materializationType( tName );
+                tName = materializationType.c_str();
+            }
             inst = reg->ObjCreate( tName, sName );
             break;
     }
@@ -319,7 +492,18 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg
             inst->AddP21Comment( comment );
         }
         assert( inst->eDesc );
-        _file.seekg( begin );
+        _file.seekg( static_cast<std::streamoff>( begin ) );
+        if( !header ) {
+            if( findNormalString( "=" ) == std::streampos( -1 ) ||
+                    !skipTokenSeparators() ) {
+                delete inst;
+                return 0;
+            }
+            if( _file.peek() == '&' && ( !skipScope() || !skipTokenSeparators() ) ) {
+                delete inst;
+                return 0;
+            }
+        }
         findNormalString( "(" );
         _file.seekg( _file.tellg() - std::streampos(1) );
         sev = inst->STEPread( instance, 0, _lazyFile->getInstMgr()->getAdapter(), _file, sName, true, false );
@@ -360,7 +544,6 @@ STEPcomplex * sectionReader::CreateSubSuperInstance( const Registry * reg, insta
     //TODO still need the schema name
     STEPcomplex * sc = new STEPcomplex( ( const_cast<Registry *>( reg ) ), names, ( int ) fileid /*, schnm*/ );
     delete[] names;
-    //TODO also delete contents of typeNames!
+    for( int i = 0; i < s; i++ ) delete typeNames[i];
     return sc;
 }
-
diff --git a/src/cllazyfile/test/lazy_index.exp b/src/cllazyfile/test/lazy_index.exp
new file mode 100644
index 000000000..241a88730
--- /dev/null
+++ b/src/cllazyfile/test/lazy_index.exp
@@ -0,0 +1,25 @@
+SCHEMA lazy_test_schema;
+
+ENTITY b;
+END_ENTITY;
+
+ENTITY a;
+    opt : OPTIONAL STRING;
+    label : STRING;
+    ref : b;
+END_ENTITY;
+
+ENTITY c;
+END_ENTITY;
+
+ENTITY d
+    SUBTYPE OF (c);
+    ref : a;
+END_ENTITY;
+
+ENTITY e;
+    ref : d;
+    opt : OPTIONAL STRING;
+END_ENTITY;
+
+END_SCHEMA;
diff --git a/src/cllazyfile/test/lazy_index.stp b/src/cllazyfile/test/lazy_index.stp
new file mode 100644
index 000000000..7d3cc8629
--- /dev/null
+++ b/src/cllazyfile/test/lazy_index.stp
@@ -0,0 +1,17 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('lazy index test'),'2;1');
+FILE_NAME('lazy_index.stp','2026-07-17T00:00:00',('STEPcode'),('STEPcode'),'','','');
+FILE_SCHEMA(('LAZY_TEST_SCHEMA { 1 0 10303 999 }'));
+ENDSEC;
+DATA('second',('LAZY_TEST_SCHEMA'));
+#1=A($,'escaped quote '' and unicode \X2\03A9\X0\',#2);
+#2=B();
+ENDSEC;
+/* A second data section must not be consumed by lookahead. */
+DATA;
+#3=(C()D(#1));
+#4=E(#3,$);
+#5=E(#99,$);
+ENDSEC;
+END-ISO-10303-21;
diff --git a/src/cllazyfile/test/lazy_scope.stp b/src/cllazyfile/test/lazy_scope.stp
new file mode 100644
index 000000000..fdff89c40
--- /dev/null
+++ b/src/cllazyfile/test/lazy_scope.stp
@@ -0,0 +1,17 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('lazy scope test'),'1;1');
+FILE_NAME('lazy_scope.stp','2026-07-31T00:00:00',('STEPcode'),('STEPcode'),'','','');
+FILE_SCHEMA(('LAZY_TEST_SCHEMA { 1 0 10303 999 }'));
+ENDSEC;
+DATA;
+#10=&SCOPE
+#11=&SCOPE
+#12=B();
+ENDSCOPE /#12/ A($,'nested owner',#12);
+ENDSCOPE /#11/ A($,'outer owner',#12);
+#20=&SCOPE
+#21=A($,'complex child',#12);
+ENDSCOPE (C()D(#21));
+ENDSEC;
+END-ISO-10303-21;