Fix streaming mode

cramming/backend/torch_default.py

@@ -41,7 +41,7 @@ def initialize_torch(model, dataset, tokenizer, cfg_train, cfg_impl, setup=_defa
         model_engine = TorchEngineFull(model, cfg_train, cfg_impl, setup=setup, seq_length=tokenizer.model_max_length)
     else:
         model_engine = TorchEngineMinimal(model, cfg_train, cfg_impl, setup=setup, seq_length=tokenizer.model_max_length)
-    model_engine.train()
+    model_engine.train()  # This is the default engine state. Pretraining scripts may change this.
     return model_engine, model_engine.optimizer, model_engine.scheduler, dataloader
 
 

cramming/config/data/roots-readymade.yaml → cramming/config/data/pile-readymade.yaml

@@ -1,10 +1,11 @@
 # Draw a preprocessed dataset directly from my HF profile.
 # This dataset is already tokenized, you "have" to load the correct tokenizer
-name: roots-mini
+name: pile-readymade
 sources:
   hub:
     provider: hub
-hf_location: JonasGeiping/roots-mini_WordPiecex32768_2a22a1d08cbdc9685c3c795938ebebfb
+hf_location: JonasGeiping/the_pile_WordPiecex32768_97b8e776baafb99c3892e6572a9f51b3
 streaming: True
 
 vocab_size: 32768 # cannot be changed!
+seq_length: 128 # cannot be changed!

cramming/data/pretraining_preparation.py

@@ -470,5 +470,5 @@ def _load_from_hub(cfg_data, data_path):
             repo_type="dataset",
             local_dir=os.path.join(data_path),
         )
-    tokenizer = load_tokenizer(os.path.join(data_path, "tokenizer"), cache_dir=data_path)
+    tokenizer = load_tokenizer(os.path.join(data_path, "tokenizer"), seq_length=cfg_data.seq_length, cache_dir=data_path)
     return tokenized_dataset, tokenizer

load_local_model.py

@@ -38,6 +38,7 @@ def main_load_process(cfg, setup):
         # Save to hub
         if cfg.impl.push_to_huggingface_hub:
             model_engine.push_to_hub(tokenizer, cfg, dryrun=cfg.dryrun)
+    return {}
 
 
 @hydra.main(config_path="cramming/config", config_name="cfg_pretrain", version_base="1.1")

upload_processed_dataset.py

@@ -75,6 +75,7 @@ def upload(cfg, setup):
                 repo_type="dataset",
             )
         log.info("Upload completed succesfully.")
+    return {}
 
 
 @hydra.main(config_path="cramming/config", config_name="cfg_pretrain", version_base="1.1")