Skip to content

Commit

Permalink
unifying with supersnippets
Browse files Browse the repository at this point in the history
  • Loading branch information
@sole
sole committed Aug 21, 2010
1 parent 71a6130 commit dd4f391
Show file tree
Hide file tree
Showing 7 changed files with 523 additions and 0 deletions.
10 changes: 10 additions & 0 deletions README
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ Old style 'table' of contents:
|
+-- wav2ogg - encode all .wav files in current directory to .ogg
|
+ database
|
+-- db_import - php+mysql tool for renaming/importing database tables and field names
|
+-- tables2utf8 - SQL for converting badly encoded utf8 data into real utf8 data (for WordPress but can be adapted to other applications)
|
+ gimp
|
+-- generate_bitmap_font - gimp plug-in
|
+ psp
|
+-- log_buttons - dump button presses and other events (uses SDL)
Expand Down
20 changes: 20 additions & 0 deletions database/db_import/config.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<?php
$config = array(
"database" => array(
'source' => array( 'driver' => 'mysql',
'connect' => 'mysql_connect',
'host' => 'localhost',
'login' => 'sole',
'password' => 'notocar',
'database' => 'jahnet_v2',
'prefix' => ''),
'destination' => array( 'driver' => 'mysql',
'connect' => 'mysql_connect',
'host' => 'localhost',
'login' => 'sole',
'password' => 'notocar',
'database' => 'jahnet_cake',
'prefix' => '')
)
);
?>
215 changes: 215 additions & 0 deletions database/db_import/db_import.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
<html>
<head><title>migrate!</title></head>
<body>
<style type="text/css">.error{font-weight: bold; color:#900;}</style>
<?php
require_once('config.php');
require_once('tables.php');

$t_start = time();

$db_src = new Database($config['database']['source']);
$db_dst = new Database($config['database']['destination']);

foreach($tables as $table) {
$src_fields = $db_src->get_table_fields($table['source']);
$excluded = isset($table['exclude']) ? explode(",", $table['exclude']) : array();
$included = isset($table['include']) ? explode(",", $table['include']) : array();
if(count($excluded)>0 && count($included)>0) {
ER("Error in $table: both included and excluded fields at the same time. Skipping.");
continue;
} else if(count($excluded)>0) {
$mode = "excluding";
} else if(count($included)>0) {
$mode = "including";
}

$dst_fields = array();
foreach($src_fields as $name => $field) {
if($mode=="excluding" && in_array($name, $excluded)) {
continue;
} else if($mode=="including" && !in_array($name, $included)) {
continue;
}
if(isset($table['map']) && array_key_exists($name, $table['map'])) {
$dst_name = $table['map'][$name];
$field['src_name'] = $name;
} else {
$dst_name = $name;
}
$dst_fields[$dst_name] = $field;
}

$table_info =
array(
"name" => $table['destination'],
"fields" => $dst_fields
);

$update_mode = $db_dst->table_exists($table_info['name']);
$res = $db_dst->process_table($table_info);
if($res!=false) {
$first_field = array_shift(array_keys($src_fields));
$query = "SELECT * FROM {$db_src->name}.$table[source] ORDER BY $first_field ASC";
$res = mysql_query($query, $db_src->dbh);
while($row = mysql_fetch_assoc($res)) {
if(!$update_mode) {
$q_insert = "INSERT INTO {$db_dst->name}.$table[destination] ";
$pairs = get_field_pairs($dst_fields, $row);
$columns = implode(",", array_keys($pairs));
$values = implode(",", array_values($pairs));
$q_insert .= "($columns) VALUES ($values);";
$res_insert = mysql_query($q_insert);
if(!$res_insert) {
ER($res_insert.": ".mysql_error());
}
} else {
$q_update = "UPDATE {$db_dst->name}.$table[destination] SET ";
$pairs = get_field_pairs($dst_fields, $row);
$sql_pairs = array();
foreach($pairs as $field=>$value) {
$sql_pairs[] = " $field = $value";
}
$q_update.= implode(", ", $sql_pairs);
$q_update.= " WHERE id='".$row[$table['use_key']]."'";
$res_update = mysql_query($q_update);
if(!$res_update) {
ER($res_update.": ".mysql_error());
}
}
}
}
}

$t_total = time() - $t_start;
echo "$t_total seconds!!";

function get_field_pairs($dst_fields, $row) {
$pairs = array();
foreach($dst_fields as $name=>$field) {
if(isset($row[$name])) {
$value = "'".addslashes($row[$name])."'";
} else if (isset($row[$field['src_name']])) {
$value = "'".addslashes($row[$field['src_name']])."'";
} else {
continue;
}
$pairs["`$name`"] = $value;
}
return $pairs;
}

class Database {
var $dbh;
var $name;

function Database($db_config) {
$f = $db_config['connect'];
$this->dbh = $f($db_config['server'], $db_config['login'], $db_config['password']);
$this->name = $db_config['database'];
}

function get_table_fields($table) {
// do a show fields, return associative array
$query = "SHOW FIELDS FROM {$this->name}.$table";
$res = mysql_query($query, $this->dbh);
$fields = array();
while($row = mysql_fetch_assoc($res)) {
$name = $row['Field'];
$fields[$name]['type'] = $row['Type'];
$fields[$name]['null'] = $row['Null']=="YES" ? true : false;
$fields[$name]['key'] = !empty($row['Key']) ? $row['Key'] : null;
$fields[$name]['default'] = isset($row['Default']) ? $row['Default'] : null;
$fields[$name]['extra'] = !empty($row['Extra']) ? $row['Extra'] : null;
}
return $fields;
}

/**
* take care of creating or updating the table structure (depending on the table previously existing or not)
*/
function process_table($table_info) {
print_r($table_info);echo "<hr />";
if($this->table_exists($table_info['name'])) {
$query = "ALTER TABLE {$this->name}.`$table_info[name]`";
} else {
$query = "CREATE TABLE {$this->name}.`$table_info[name]` (";
$create = true;
}

$columns = array();
$keys = array();
foreach($table_info['fields'] as $name => $field) {
$column = "\n\t";
if(!$create) {
$column .= "ADD COLUMN ";
}
$column .= "`$name` $field[type]";
if(!$field['null'])
$column .= " NOT NULL";
if(!is_null($field['extra']))
$column .= " $field[extra]";
if(!is_null($field['default']))
$column .= " default '$field[default]'";
else
$column .= " default NULL";
$columns [] = $column;

if(isset($field['key'])) {
$keys[$field['key']][] = $name;
}
}

$query .= implode(",", $columns);

$constraints = array();
foreach($keys as $key=>$keyfields) {
for($i=0; $i<count($keyfields); $i++) {
$keyfields[$i] = "`".$keyfields[$i]."`";
}
$keytext = implode(",", $keyfields);
if($key=="PRI") {
$constraints[$key] = "PRIMARY KEY($keytext)";
} else if($key=="UNI") {
$constraints[$key] = "UNIQUE($keytext)";
}
if(!$create) {
$constraints[$key] = "ADD ".$constraints[$key];
}
}
if(count($constraints)>0) {
$query.= ",\n".implode(",\n", $constraints);
}

if($create) {
$query .="\n)";
}
echo "<pre>$query</pre>";
$res = mysql_query($query, $this->dbh);
if($res == false) {
ER(mysql_error());
} else {
OK("processed");
}
return $res;
}

function table_exists($table) {
if (mysql_query("SELECT 1 FROM ".$this->name.".`".$table."` LIMIT 0", $this->dbh)) {
return true;
} else {
return false;
}
}
}

function OK($text) {
echo "<p>$text</p>";
}

function ER($text) {
echo "<p class=\"error\">$text</p>";
}
?>
</body>
</html>
82 changes: 82 additions & 0 deletions database/db_import/readme.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
Script for importing and transforming databases
-----------------------------------------------

1. Configuration
2. Use it!


1. Configuration
=================

There are two files you need to alter appropiately:

a) config.php

There's the definition of the source and destination databases. The syntax is the same than cake php's one.

You can also use the same database as source and destination - but horrible things may happen if there are overlappings between table names. This has not been tested.

b) tables.php

Here you specify the tables you want to move, with the modifications required:

Create an entry in $tables for each table in the source database that you want to process. This is the meaning of each parameter:

- source: table name in the source database

- destination: table name in the destination database

- map: array containing field names in the source database and their correspondence in the destination one. This is how field names are renamed. For example, you can map fname to first_name with $map = array('fname' => 'first_name');

- exclude: comma separated list of fields you do not want to include when copying the table
- include: comma separated list of fields you want to include when copying the table

Please note that you can't specify exclude and include at the same time. Doing so will result in the table being skipped.

The way it works depends on if 'exclude' or 'include' are specified.
If 'exclude' is specified, the script will include all the fields excluding the specified ones.
If 'include' is specified, the script will include only the specified fields.

For example, if you need to discard more fields than the ones you want to keep, you'll use only include, avoiding a long "exclude" list.

- use_key: this is only used when you need to add data from one table into an existing one, i.e. merging two tables. Specify with this the name of the source table which represents the "id" in the destination table.

Example:

You have two tables that you want to merge in just one, user and user_details:
user (user_id, name, firstname, etc...)
user_details (user_id, homepage, motto, etc...)


In this case $tables will look like this ([...] means there's more irrelevant data:

$tables[] =
array(
'source' => 'user',
'destination'=> 'users',
'map' =>
array(
"user_id" => "id",
"firstname" => "first_name",
"lastname" => "last_name"
),
'exclude' => "type,flagged [...]"
);

$tables[] =
array(
'source' => 'user_details',
'destination'=> 'users',
'include' => "homepage",
'use_key' => 'user_id'
);

Note how the main table users is created with the data of user table, and then the data from user_details is merged into users by specifying the destination table 'users' and the key 'user_id'.

What the script does is detect the table already exist and then instead of generating series of INSERT's, it does generate UPDATE's and adding a "WHERE id = " and the value in the second table of the key you specified. In this case it would be using the value of user_details.user_id, since it's a 1:1 correspondence.

2. Use it
===========

For using this script, place all the files of the db_import folder inside the webroot - so that it can be accessed via a web browser. For example: http://localhost/db_import/db_import.php
If there are errors they will be output. In that case it may be worth to delete all the created tables, fix the errors and re-run the script again.
50 changes: 50 additions & 0 deletions database/db_import/tables.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
<?php
$tables = array();

$tables[] =
array(
'source' => 'user',
'destination'=> 'users',
'map' =>
array(
"user_id" => "id",
"firstname" => "first_name",
"lastname" => "last_name"
),
'exclude' => "type,flagged,address_id,shipping_address_id,billing_address_id,shiptobilling,from_site_id,pluginz_user_id,jahshaka_user_id,avatar,dob,upload_quota,uploaded_amount,deleted,deleted_date,frontpage"
);

$tables[] =
array(
'source' => 'user_details',
'destination'=> 'users',
'include' => "homepage,interests,movies,music,skills,role,industry,motto,loc_city,loc_country,user_description,last_updated,profile_viewed",
'use_key' => "user_id"
);

$tables[] =
array(
'source' => 'user_info',
'destination'=> 'users',
'map' =>
array(
'account_created' => "created",
'last_modified' => "modified"
),
'include' => "account_created,last_modified",
'use_key' => "user_id"
);

$tables[] =
array(
'source' => 'asset',
'destination'=> 'assets',
'map' =>
array(
"asset_id" => "id",
"lastname" => "last_name"
),
'exclude' => "GUID,folder"
);

?>
Loading

0 comments on commit dd4f391

Please sign in to comment.