Permalink
Browse files

ENHANCEMENT Added BulkLoader->deleteExistingRecords(), removed unnece…

…ssary parameters from BulkLoader->load()

ENHANCEMENT Decreased memory usage in BulkLoader->load() when deleting all records before importing
MINOR Re-enabled CsvBulkloaderTest cases, were disabled by accident

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/sapphire/trunk@94250 467b73ca-7a2a-4603-9d3b-597d59a354a9
  • Loading branch information...
1 parent 261f8f8 commit ef28a27b251b5e6d5f914d7c31fe07d32cd4f672 @chillu chillu committed Dec 2, 2009
Showing with 142 additions and 132 deletions.
  1. +19 −10 dev/BulkLoader.php
  2. +123 −122 tests/dev/CsvBulkLoaderTest.php
View
@@ -118,27 +118,36 @@
*/
public $duplicateChecks = array();
+ /**
+ * @var Boolean $clearBeforeImport Delete ALL records before importing.
+ */
+ public $deleteExistingRecords = false;
+
function __construct($objectClass) {
$this->objectClass = $objectClass;
parent::__construct();
}
/*
- * Load the given file via {@link self::processAll()} and {@link self::processRecord()}. Optionally truncates (clear) the table before it imports.
+ * Load the given file via {@link self::processAll()} and {@link self::processRecord()}.
+ * Optionally truncates (clear) the table before it imports.
*
* @return BulkLoader_Result See {@link self::processAll()}
*/
- public function load($filepath, $memory_limit='512M', $clear_table_before_import=false) {
+ public function load($filepath) {
ini_set('max_execution_time', 3600);
+ increase_memory_limit_to('512M');
- increase_memory_limit_to($memory_limit);
-
- if ($clear_table_before_import) {
- $objectSet = DataObject::get($this->objectClass); //get all instances of the to be imported data object
- if (!empty($objectSet)) {
- foreach($objectSet as $obj) {
- $obj->delete(); //deleting objects ensures that versions are also deleted (truncating would just delete the main table); performance is slower, however
- }
+ //get all instances of the to be imported data object
+ if($this->deleteExistingRecords) {
+ $q = singleton($this->objectClass)->buildSQL();
+ $q->select = array('"ID"');
+ $ids = $q->execute()->column('ID');
+ foreach($ids as $id) {
+ $obj = DataObject::get_by_id($this->objectClass, $id);
+ $obj->delete();
+ $obj->destroy();
+ unset($obj);
}
}
@@ -7,128 +7,129 @@
class CsvBulkLoaderTest extends SapphireTest {
static $fixture_file = 'sapphire/tests/dev/CsvBulkLoaderTest.yml';
- // /**
- // * Test plain import with column auto-detection
- // */
- // function testLoad() {
- // $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
- // $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
- // $file = fopen($filepath, 'r');
- // $compareCount = $this->getLineCount($file);
- // fgetcsv($file); // pop header row
- // $compareRow = fgetcsv($file);
- // $results = $loader->load($filepath);
- //
- // // Test that right amount of columns was imported
- // $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
- //
- // // Test that columns were correctly imported
- // $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
- // $this->assertNotNull($obj);
- // $this->assertEquals("He's a good guy", $obj->Biography);
- // $this->assertEquals("1988-01-31", $obj->Birthday);
- // $this->assertEquals("1", $obj->IsRegistered);
- //
- // fclose($file);
- // }
- //
- // /**
- // * Test plain import with clear_table_before_import
- // */
- // function testClearTableBeforeImport() {
- // $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
- // $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
- // $results1 = $loader->load($filepath, '512MB', false); //leave existing data there on first CSV import
- // $this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
- //
- // $results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
- // $resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them
- //
- // $this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
- // }
- //
- // /**
- // * Test import with manual column mapping
- // */
- // function testLoadWithColumnMap() {
- // $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
- // $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv';
- // $file = fopen($filepath, 'r');
- // $compareCount = $this->getLineCount($file);
- // $compareRow = fgetcsv($file);
- // $loader->columnMap = array(
- // 'FirstName',
- // 'Biography',
- // null, // ignored column
- // 'Birthday',
- // 'IsRegistered'
- // );
- // $loader->hasHeaderRow = false;
- // $results = $loader->load($filepath);
- //
- // // Test that right amount of columns was imported
- // $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
- //
- // // Test that columns were correctly imported
- // $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
- // $this->assertNotNull($obj);
- // $this->assertEquals("He's a good guy", $obj->Biography);
- // $this->assertEquals("1988-01-31", $obj->Birthday);
- // $this->assertEquals("1", $obj->IsRegistered);
- //
- // $obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
- // $this->assertNotNull($obj2);
- // $this->assertEquals('0', $obj2->IsRegistered);
- //
- // fclose($file);
- // }
- //
- // /**
- // * Test import with manual column mapping and custom column names
- // */
- // function testLoadWithCustomHeaderAndRelation() {
- // $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
- // $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
- // $file = fopen($filepath, 'r');
- // $compareCount = $this->getLineCount($file);
- // fgetcsv($file); // pop header row
- // $compareRow = fgetcsv($file);
- // $loader->columnMap = array(
- // 'first name' => 'FirstName',
- // 'bio' => 'Biography',
- // 'bday' => 'Birthday',
- // 'teamtitle' => 'Team.Title', // test existing relation
- // 'teamsize' => 'Team.TeamSize', // test existing relation
- // 'salary' => 'Contract.Amount' // test relation creation
- // );
- // $loader->hasHeaderRow = true;
- // $loader->relationCallbacks = array(
- // 'Team.Title' => array(
- // 'relationname' => 'Team',
- // 'callback' => 'getTeamByTitle'
- // ),
- // // contract should be automatically discovered
- // );
- // $results = $loader->load($filepath);
- //
- // // Test that right amount of columns was imported
- // $this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
- //
- // // Test of augumenting existing relation (created by fixture)
- // $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
- // $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
- //
- // // Test of creating relation
- // $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
- // $testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
- // $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
- //
- // // Test nested setting of relation properties
- // $contractAmount = DBField::create('Currency', $compareRow[5])->RAW();
- // $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works');
- //
- // fclose($file);
- // }
+ /**
+ * Test plain import with column auto-detection
+ */
+ function testLoad() {
+ $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
+ $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
+ $file = fopen($filepath, 'r');
+ $compareCount = $this->getLineCount($file);
+ fgetcsv($file); // pop header row
+ $compareRow = fgetcsv($file);
+ $results = $loader->load($filepath);
+
+ // Test that right amount of columns was imported
+ $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
+
+ // Test that columns were correctly imported
+ $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
+ $this->assertNotNull($obj);
+ $this->assertEquals("He's a good guy", $obj->Biography);
+ $this->assertEquals("1988-01-31", $obj->Birthday);
+ $this->assertEquals("1", $obj->IsRegistered);
+
+ fclose($file);
+ }
+
+ /**
+ * Test plain import with clear_table_before_import
+ */
+ function testDeleteExistingRecords() {
+ $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
+ $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithHeader.csv';
+ $loader->deleteExistingRecords = true;
+ $results1 = $loader->load($filepath);
+ $this->assertEquals(4, $results1->Count(), 'Test correct count of imported data on first load');
+
+ $results2 = $loader->load($filepath, '512MB', true); //delete existing data before doing second CSV import
+ $resultDataObject = DataObject::get('CsvBulkLoaderTest_Player'); //get all instances of the loaded DataObject from the database and count them
+
+ $this->assertEquals(4, $resultDataObject->Count(), 'Test if existing data is deleted before new data is added');
+ }
+
+ /**
+ * Test import with manual column mapping
+ */
+ function testLoadWithColumnMap() {
+ $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
+ $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_Players.csv';
+ $file = fopen($filepath, 'r');
+ $compareCount = $this->getLineCount($file);
+ $compareRow = fgetcsv($file);
+ $loader->columnMap = array(
+ 'FirstName',
+ 'Biography',
+ null, // ignored column
+ 'Birthday',
+ 'IsRegistered'
+ );
+ $loader->hasHeaderRow = false;
+ $results = $loader->load($filepath);
+
+ // Test that right amount of columns was imported
+ $this->assertEquals(4, $results->Count(), 'Test correct count of imported data');
+
+ // Test that columns were correctly imported
+ $obj = DataObject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
+ $this->assertNotNull($obj);
+ $this->assertEquals("He's a good guy", $obj->Biography);
+ $this->assertEquals("1988-01-31", $obj->Birthday);
+ $this->assertEquals("1", $obj->IsRegistered);
+
+ $obj2 = DataObject::get_one('CsvBulkLoaderTest_Player', "\"FirstName\" = 'Jane'");
+ $this->assertNotNull($obj2);
+ $this->assertEquals('0', $obj2->IsRegistered);
+
+ fclose($file);
+ }
+
+ /**
+ * Test import with manual column mapping and custom column names
+ */
+ function testLoadWithCustomHeaderAndRelation() {
+ $loader = new CsvBulkLoader('CsvBulkLoaderTest_Player');
+ $filepath = Director::baseFolder() . '/sapphire/tests/dev/CsvBulkLoaderTest_PlayersWithCustomHeaderAndRelation.csv';
+ $file = fopen($filepath, 'r');
+ $compareCount = $this->getLineCount($file);
+ fgetcsv($file); // pop header row
+ $compareRow = fgetcsv($file);
+ $loader->columnMap = array(
+ 'first name' => 'FirstName',
+ 'bio' => 'Biography',
+ 'bday' => 'Birthday',
+ 'teamtitle' => 'Team.Title', // test existing relation
+ 'teamsize' => 'Team.TeamSize', // test existing relation
+ 'salary' => 'Contract.Amount' // test relation creation
+ );
+ $loader->hasHeaderRow = true;
+ $loader->relationCallbacks = array(
+ 'Team.Title' => array(
+ 'relationname' => 'Team',
+ 'callback' => 'getTeamByTitle'
+ ),
+ // contract should be automatically discovered
+ );
+ $results = $loader->load($filepath);
+
+ // Test that right amount of columns was imported
+ $this->assertEquals(1, $results->Count(), 'Test correct count of imported data');
+
+ // Test of augumenting existing relation (created by fixture)
+ $testTeam = DataObject::get_one('CsvBulkLoaderTest_Team', null, null, '"Created" DESC');
+ $this->assertEquals('20', $testTeam->TeamSize, 'Augumenting existing has_one relation works');
+
+ // Test of creating relation
+ $testContract = DataObject::get_one('CsvBulkLoaderTest_PlayerContract');
+ $testPlayer = Dataobject::get_one("CsvBulkLoaderTest_Player", "\"FirstName\" = 'John'");
+ $this->assertEquals($testPlayer->ContractID, $testContract->ID, 'Creating new has_one relation works');
+
+ // Test nested setting of relation properties
+ $contractAmount = DBField::create('Currency', $compareRow[5])->RAW();
+ $this->assertEquals($testPlayer->Contract()->Amount, $contractAmount, 'Setting nested values in a relation works');
+
+ fclose($file);
+ }
/**
* Test import with custom identifiers by importing the data.

0 comments on commit ef28a27

Please sign in to comment.