From 67661630abc1ebe733af86acd1d804090f422740 Mon Sep 17 00:00:00 2001
From: Jan Schejbal <jan-bugreport@gmx.de>
Date: Thu, 24 May 2012 03:07:52 +0200
Subject: [PATCH] Add Export/Import scripts (tested, but need
 self-review/review)

---
 export/piratenid-export.php                   | 163 +++++++++++++++++
 .../piratenid-import.php                      | 167 +++++++++---------
 export/piratenid-verify.php                   |  36 ++++
 3 files changed, 279 insertions(+), 87 deletions(-)
 create mode 100644 export/piratenid-export.php
 rename server/includes/tokenimport.inc.php => export/piratenid-import.php (60%)
 create mode 100644 export/piratenid-verify.php

diff --git a/export/piratenid-export.php b/export/piratenid-export.php
new file mode 100644
index 0000000..8e7c774
--- /dev/null
+++ b/export/piratenid-export.php
@@ -0,0 +1,163 @@
+<?php
+
+// TODO: Self-review, Fremd-review
+
+// This file needs to be placed on the system doing the export and requires piratenid-verify.php
+// Remember to update the secret in both this file and piratenid-import.php on the server receiving the import!
+
+//   USE ONLY ON A PROTECTED, INTERNAL NETWORK!
+//   The encryption is an additional security feature. It does NOT prevent against replay attacks!
+//   (i.e. if an attacker get a copy of a valid request, AND can send data to the import script,
+//   he can send the same data again and the importer will import the old data)
+
+
+/// CONFIG ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Shared secret for encrypting and authenticating token imports - needs to match secret in piratenid-import.php
+// If compromised, replace with new random value in export and import file and re-import token table
+$SECRET = "7EbkyTL7N0npJhc4Gv2oXvm4mhDyYXk8cTMg2fa1bcOiiun3Xh7l5YsNNqw0";
+
+// Source Excel file for export data (full path, remember that backslashes need to be doubled)
+$SOURCEFILE = 'C:\\Users\\Jan\\Documents\\Projekte\\piratenpartei\\piratenid\\export\\piratenidtest.xlsx';
+
+// maximum age of the source file in seconds
+$MAXAGE = 10*60*60;
+
+// URL of the piratenid-import.php script that should receive the export data (internal network only!)
+$TARGETURL = 'http://127.0.0.1:80/testimport.php'; 
+
+/// END OF CONFIG ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
+require_once('piratenid-verify.php');
+$statLVs = PiratenIDImport_getLVs();
+
+// make sure errors are fatal
+error_reporting(E_ALL & E_STRICT);
+function fatalErrors($errno, $errstr) { PiratenIDImport_err("Fehler $errno:\n$errstr\n"); }
+set_error_handler("fatalErrors");
+
+
+// check source file
+if (!is_file($SOURCEFILE)) PiratenIDImport_err('missing source file');
+if (filemtime($SOURCEFILE) < (time() - $MAXAGE)) PiratenIDImport_err('source data too old');
+
+
+// Fetch table name
+$odbc = odbc_connect("Driver={Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)};DBQ=".$SOURCEFILE, "", "") or PiratenIDImport_err("Could not open data source");
+$tablelist = odbc_tables($odbc) or PiratenIDImport_err("Could not query tables");
+$tablearr = odbc_fetch_array($tablelist) or PiratenIDImport_err("Could not get table info");
+odbc_close($odbc);
+if (empty($tablearr['TABLE_NAME'])) PiratenIDImport_err("Table name not found");
+$tablename = $tablearr['TABLE_NAME'];
+
+
+// Fetch data
+$pdo = new PDO("odbc:Driver={Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)};DBQ=".$SOURCEFILE) or PiratenIDImport_err("Could not connect to data source");
+// String concatenation in SQL and no way around it... strict checking of the table name first!
+if (!preg_match('/^[a-zA-Z0-9_-]{1,28}\\$$/D', $tablename)) PiratenIDImport_err("Invalid table name");
+$statement = $pdo->prepare('SELECT * FROM ['.$tablename.'] ORDER BY user_token') or PiratenIDImport_err("Could not prepare query");
+if ($statement->execute()) {
+	$input_data = $statement->fetchAll(PDO::FETCH_ASSOC);
+} else {
+	$err = $statement->errorInfo();
+	PiratenIDImport_err("Could not fetch data: " . $err[2]);
+}
+
+if (empty($input_data)) PiratenIDImport_err("Could not fetch data - empty data set received");
+if (empty($input_data)) PiratenIDImport_err("Could not fetch data - empty data set received");
+if (count($input_data) < 2) die("Invalid data: less than 2 entries");
+
+// Convert data
+$output_data = array();
+$count_total = array();
+$count_stimmberechtigt = array();
+
+foreach ($statLVs as $lv) {
+	$count_total[$lv] = 0;
+	$count_stimmberechtigt[$lv] = 0;
+}
+
+foreach ($input_data as $entry) {
+	if (empty($entry['user_token'])) PiratenIDImport_err("Missing field: user_token");
+	if (empty($entry['USER_LV'])) PiratenIDImport_err("Missing field: USER_LV");
+	if (empty($entry['USER_Stimmberechtigt'])) PiratenIDImport_err("Missing field: USER_Stimmberechtigt");
+	
+	$token = $entry['user_token'];
+	$mitgliedschaft_bund   = 'ja';
+	$mitgliedschaft_land   = $entry['USER_LV'];
+	$mitgliedschaft_kreis  = '';
+	$mitgliedschaft_bezirk = '';
+	$mitgliedschaft_ort    = '';
+	$stimmberechtigt       = (($entry['USER_Stimmberechtigt'] == -1) ? 'ja' : 'nein');
+	$out_entry = array($token, $mitgliedschaft_bund, $mitgliedschaft_land, $mitgliedschaft_kreis, $mitgliedschaft_bezirk, $mitgliedschaft_ort, $stimmberechtigt);
+	PiratenIDImport_verifyEntry($out_entry);
+	$output_data[] = $out_entry;
+	
+	$count_total[$mitgliedschaft_land]++;
+	if ($stimmberechtigt === 'ja') $count_stimmberechtigt[$mitgliedschaft_land]++;
+}
+
+unset($input_data); // conserve memory
+
+$json = json_encode($output_data);
+unset($output_data); // conserve memory
+
+// Derive keys
+$key_crypto_raw = hash('sha256', 'crypto|'.$SECRET, true); // encryption key
+$key_hmac_raw = hash('sha256', 'hmac|'.$SECRET, true);     // HMAC integrity key
+$key_auth_raw = hash('sha256', 'auth|'.$SECRET, true);    // Auth token
+if (strlen($key_crypto_raw) != 32 || strlen($key_hmac_raw) != 32 || strlen($key_auth_raw) != 32 ) PiratenIDImport_err("Key derivation failed");
+
+
+// generate secure IV
+$strong = false;
+$iv_raw = openssl_random_pseudo_bytes(16, $strong);
+if (strlen($iv_raw) != 16 || !$strong) PiratenIDImport_err("IV generation failed");
+
+// Calculate HMAC
+$hmac_hex = hash_hmac('sha256', $json, $key_hmac_raw, false);
+if (strlen($hmac_hex) != 64) PiratenIDImport_err("HMAC calculation failed");
+
+// Encrypt
+$encrypted = openssl_encrypt($hmac_hex.$json, 'aes-256-cbc' , $key_crypto_raw, true, $iv_raw);
+if ($encrypted === false) PiratenIDImport_err("Encryption failed");
+unset($json); // conserve memory
+
+// Send
+
+$context = stream_context_create(
+			array(
+				'http' =>
+					array(
+						'method'  => 'POST',
+						'ignore_errors' => true,
+						'header' => 'Content-type: application/octet-stream',
+						'content' => $key_auth_raw.$iv_raw.$encrypted,
+						'timeout'  => 30
+					)
+			)
+		);
+
+$result = file_get_contents($TARGETURL, false, $context);
+
+echo "Server response:\n-------------------------------\n";
+echo $result;
+echo "\n-------------------------------\n";
+
+if ($result === "Import successful") {
+	echo "Looks like the import was successful!\n\n";
+	echo "Stats:\n";
+	foreach ($statLVs as $lv) {
+		$statLVstr = $lv;
+		if ($statLVstr === '') $statLVstr = 'XX';
+		printf(" | $statLVstr = %6d | $statLVstr-stimmberechtigt = %6d\n", $count_total[$lv], $count_stimmberechtigt[$lv]);
+	}
+} else {
+	echo "Looks like the import failed!";
+	exit(1);
+}
diff --git a/server/includes/tokenimport.inc.php b/export/piratenid-import.php
similarity index 60%
rename from server/includes/tokenimport.inc.php
rename to export/piratenid-import.php
index 9ca318f..bee2ea6 100644
--- a/server/includes/tokenimport.inc.php
+++ b/export/piratenid-import.php
@@ -1,45 +1,63 @@
-<?PHP
+<?php
 
-die("DO NOT USE - contains totally untested functions. import will probably have to be done differently anyways");
+// TODO: Self-review, Fremd-review
 
+// This file needs to be placed on the PiratenID server and requires piratenid-verify.php
+// It should be reachable only from the server doing the export
+// Suggestion: Deploy as a separate site, listening on a separate port
 
 
-/*
+/// CONFIG ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-This code needs the following additions to siteconstants.inc.php - WARNING, CSV/POST IMPORT UNTESTED!
-	
-#########################################################################################################################	
-// SHA256 hash of key required to perform token imports.
-// If key is compromised, replace with new random value and re-import token table
-$tokenimport_key = "6b1382dd00b77cd2dd40688f03f7093753be09ab88ad5f23171fe867144bdf97"; // Jzbj8CfJ1nGLZxAByRca
-
-// Key for authenticating token imports using HMAC
-// If key is compromised, replace with new random value and re-import token table
-$tokenimport_hmac_key = "adhzbxHFTInvMM7zn8jBxfI9VR5moVRzX6Yj5tpIB3piBNHPy3adIeUq1Cq7";
+// Shared secret for encrypting and authenticating token imports - needs to match secret in piratenid-export.php
+// If compromised, replace with new random value in export and import file and re-import token table
+$SECRET = "7EbkyTL7N0npJhc4Gv2oXvm4mhDyYXk8cTMg2fa1bcOiiun3Xh7l5YsNNqw0";
 
-// IP from which tokens will be imported. Only this IP will be allowed to request token imports
-$tokenimport_ip = "127.0.0.1";
+$ALLOWED_IP = '127.0.0.1'; // IP from which tokens will be imported. Only this IP will be allowed to request token imports
 
-// Database login data for token import - change if regular DB user has insufficient permissions
-function getDatabaseImportPDO() {
-	return getDatabasePDO();
+function getDatabaseImportPDO() { // Database login data for token import
+	return new PDO('mysql:dbname=piratenid;host=127.0.0.1', "root", "");
 }
 
-#########################################################################################################################	
+/// END OF CONFIG ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 
-*/
-
 
 
 
 
 
+error_reporting(E_ALL & E_STRICT);
+function fatalErrors($errno, $errstr) { die("Fehler $errno:\n$errstr\n"); }
+set_error_handler("fatalErrors");
 
 
-/*
-	Diese Datei bietet eine Funktion "PiratenIDImport_import" für den Datenimport in PiratenID.
+// verifies an entry (single row)
+// function "err($errormessage)" needs to be defined!
+function PiratenIDImport_verifyEntry($entry) {
+	if (!is_array($entry)) die("Invalid data: entry not an array");
+	if (count($entry) != 7) die("Invalid data: wrong number of values");
+	for ($i = 0; $i<7; $i++) {
+		if (!is_string($entry[$i])) die("Invalid data: value not a string");
+		if (strlen($entry[$i]) > 100) die("Invalid data: value too long");
+		if (strpos($entry[$i], "\xC3\x83") !== false) die("Invalid data: looks like double UTF-8 encoding");
+		if (!mb_detect_encoding($entry[$i], 'UTF-8', true)) die("Invalid data: value not UTF-8");
+		if (!mb_check_encoding($entry[$i], 'UTF-8')) die("Invalid data: invalid UTF-8 sequence");
+		
+	}
+	
+	if (!preg_match('/^[a-f0-9]{64}$/D', $entry[0])) die("Invalid data: invalid token value");
+	if ($entry[0] == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') die("Invalid data: Token is hash of empty string");
 	
+	
+	if ($entry[1] !== 'ja' && $entry[1] !== 'nein') die("Invalid data: mitgliedschaft-bund must be either 'ja' or 'nein'");
+	if (!in_array($entry[2], array('', 'BW', 'BY', 'BE', 'BB', 'HB', 'HH', 'HE', 'MV', 'NI', 'NW', 'RP', 'SL', 'SN', 'ST', 'SH', 'TH') ,true)) {
+		die("Invalid data: value for mitgliedschaft-land is not in whitelist");
+	}
+	if ($entry[6] !== 'ja' && $entry[6] !== 'nein' && $entry[6] !== '') die("Invalid data: stimmberechtigt must be 'ja', 'nein' or empty string");
+}
+
+/*	
 	PiratenIDImport_import($db, $dataarray, $ignorelength = false): Importiert Token-Daten in PiratenID
 		$db: Datenbank-PDO mit ausreichenden Zugriffsrechten
 		$dataarray: Die zu importierenden Daten als array von arrays. Jedes sub-array hat folgendes Format:
@@ -65,32 +83,11 @@ function getDatabaseImportPDO() {
 			Ist dieser Wert false oder nicht gesetzt, wird ein Import abgelehnt, wenn weniger als 1000 Datensätze geliefert werden.
 			Dies soll verhindern, dass durch einen defekten Import die Token-Datenbank gelöscht wird.
 */
-
-// verifies an entry (single row)
-function PiratenIDImport_verifyEntry($entry) {
-	if (!is_array($entry)) die("Invalid data: entry not an array");
-	if (count($entry) != 7) die("Invalid data: wrong number of values");
-	for ($i = 0; $i<7; $i++) {
-		if (!is_string($entry[$i])) die("Invalid data: value not a string");
-		if (strlen($entry[$i]) > 100) die("Invalid data: value too long");
-		if (strpos($entry[$i], "\xC3\x83") !== false) die("Invalid data: looks like double UTF-8 encoding");
-		if (!mb_detect_encoding($entry[$i], 'UTF-8', true)) die("Invalid data: value not UTF-8");
-		if (!mb_check_encoding($entry[$i], 'UTF-8')) die("Invalid data: invalid UTF-8 sequence");
-		
-	}
-	
-	if (!preg_match('/^[a-f0-9]{64}$/D', $entry[0])) die("Invalid data: invalid token value");
-	if ($entry[0] == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') die("Invalid data: Token is hash of empty string");
-	
-	if ($entry[1] !== 'ja' && $entry[1] !== 'nein') die("Invalid data: mitgliedschaft-bund must be either 'ja' or 'nein'");
-	if ($entry[6] !== 'ja' && $entry[6] !== 'nein' && $entry[6] !== '') die("Invalid data: stimmberechtigt must be 'ja', 'nein' or empty string");
-}
-
-// see top for doc
 function PiratenIDImport_import($db, $dataarray, $ignorelength = false) {
 	if (!$db) die("No database connection");
 	if (!is_array($dataarray)) die("Invalid data: data not an array");
 	if (empty($dataarray)) die("Invalid data: data is empty");
+	if (count($dataarray) < 2) die("Invalid data: less than 2 entries");
 	if (!$ignorelength && count($dataarray) < 1000) die("Invalid data: less than 1000 entries, data probably incomplete");
 	
 	$seenTokens = array();
@@ -121,54 +118,50 @@ function PiratenIDImport_import($db, $dataarray, $ignorelength = false) {
 	} catch (PDOException $e) {
 		die('Database error: ' . $e->getMessage()); // automatic rollback
 	}
-
-}
-
-
-// completely untested and abandoned for now, as this is probably not a realistic way to do imports
-function PiratenIDImport_importCSV($csv) {
-	die("abandoned function");
-	$csv = str_replace("\r\n","\n",$csv); // normalize line endings
-	$lines = explode("\n", $csv);
-	$data = array();
-	foreach ($lines as $line) {
-		if (empty($line)) continue;
-		$cells = explode("\t", $line);
-		$data[] = $cells;
-	}
-	PiratenIDImport_import($data);
 }
 
-// completely untested and abandoned for now, as this is probably not a realistic way to do imports
-function PiratenIDImport_importFromPost() {
-	die("abandoned function");
-	global $tokenimport_key;
-	global $tokenimport_hmac_key;
-	global $tokenimport_ip;
-	if (empty($tokenimport_key)) die("Server misconfiguration: Token import key not set");
-	if (empty($tokenimport_hmac_key)) die("Server misconfiguration: Token import HMAC key not set");
-	if (empty($tokenimport_ip)) die("Server misconfiguration: Token import IP not set");
+function PiratenIDImport_importFromPost($db, $ignorelength = false) {
+	global $SECRET;
+	global $ALLOWED_IP;
+	if (empty($SECRET)) die("Server misconfiguration: Token import secret not set");
+	if (empty($ALLOWED_IP)) die("Server misconfiguration: Token import IP not set");
 	
+	if (empty($_SERVER['REQUEST_METHOD'])) die("Cannot be used from command line");
 	if ($_SERVER['REQUEST_METHOD'] !== "POST") die("Must use POST");
-	if ($_SERVER['REMOTE_ADDR'] !== $tokenimport_ip) die("IP not authorized for import");
-	if (empty($_POST['key'])) die("No import key provided");
-	if (empty($_POST['data'])) die("No import data provided");
-	if (empty($_POST['hmac'])) die("No import HMAC provided");
-	$keyhash = hash('sha256',$_POST['key']);
-	if (strlen($keyhash) != 64) die("Hash function failed");
-	if ($keyhash === $tokenimport_key) {
-		$hmac = hash_hmac('sha256', $_POST['data'], $tokenimport_hmac_key);
-		if (strlen($hmac) != 64) die("Hash function (HMAC) failed");
-		if ($_POST['hmac'] === $hmac) {
-			PiratenIDImport_importCSV($_POST['data']);
-		} else {
-			die("HMAC authentication incorrect");
-		}
-	} else {
-		die("Incorrect import key");
-	}
+	if ($_SERVER['REMOTE_ADDR'] !== $ALLOWED_IP) die("IP not authorized for import");
+	
+	$postdata = file_get_contents('php://input');
+	if (strlen($postdata) < 50) die("Invalid data (too short)");
+	
+	$auth = substr($postdata, 0,32);
+	$iv = substr($postdata, 32,16);
+	$encrypted = substr($postdata, 48);
+	unset($postdata);
+
+	$key_crypto_raw = hash('sha256', 'crypto|'.$SECRET, true); // encryption key
+	$key_hmac_raw = hash('sha256', 'hmac|'.$SECRET, true);     // HMAC integrity key
+	$key_auth_raw = hash('sha256', 'auth|'.$SECRET, true);    // Authentication token
+	if (strlen($key_crypto_raw) != 32 || strlen($key_hmac_raw) != 32 || strlen($key_auth_raw) != 32 ) die("Key derivation failed");
+	
+	if ($key_auth_raw !== $auth) die("Invalid authorization");
+	
+	$decrypted = openssl_decrypt($encrypted, 'aes-256-cbc' , $key_crypto_raw, true, $iv);
+	if ($decrypted === false) die("Decryption failed");
+	unset($encrypted); // conserve memory
+	
+	$hmac = substr($decrypted, 0, 64);
+	$json = substr($decrypted, 64);
+	if (!$json) die("Parsing failed");
+	unset($decrypted); // conserve memory
+	
+	if ($hmac != hash_hmac('sha256', $json, $key_hmac_raw)) die("Wrong HMAC authentication value");
+	
+	$data = json_decode($json);
+	PiratenIDImport_import($db, $data, $ignorelength);
+	echo "Import successful";
 }
 
+PiratenIDImport_importFromPost(getDatabaseImportPDO(), true); // TODO Remove "true" for production use!
 
 /* Example:
 
diff --git a/export/piratenid-verify.php b/export/piratenid-verify.php
new file mode 100644
index 0000000..13103a2
--- /dev/null
+++ b/export/piratenid-verify.php
@@ -0,0 +1,36 @@
+<?php
+
+// Replacement for die() which sets the exit code (ERRORLEVEL on win)
+function PiratenIDImport_err($str) {
+	echo $str;
+	exit(1);
+}
+
+function PiratenIDImport_getLVs() {
+	return array('BW', 'BY', 'BE', 'BB', 'HB', 'HH', 'HE', 'MV', 'NI', 'NW', 'RP', 'SL', 'SN', 'ST', 'SH', 'TH', '');
+}
+
+// verifies an entry (single row)
+function PiratenIDImport_verifyEntry($entry) {
+	$LVs = PiratenIDImport_getLVs();
+	if (!is_array($entry)) PiratenIDImport_err("Invalid data: entry not an array");
+	if (count($entry) != 7) PiratenIDImport_err("Invalid data: wrong number of values");
+	for ($i = 0; $i<7; $i++) {
+		if (!is_string($entry[$i])) PiratenIDImport_err("Invalid data: value not a string");
+		if (strlen($entry[$i]) > 100) PiratenIDImport_err("Invalid data: value too long");
+		if (strpos($entry[$i], "\xC3\x83") !== false) PiratenIDImport_err("Invalid data: looks like double UTF-8 encoding");
+		if (!mb_detect_encoding($entry[$i], 'UTF-8', true)) PiratenIDImport_err("Invalid data: value not UTF-8");
+		if (!mb_check_encoding($entry[$i], 'UTF-8')) PiratenIDImport_err("Invalid data: invalid UTF-8 sequence");
+		
+	}
+	
+	if (!preg_match('/^[a-f0-9]{64}$/D', $entry[0])) PiratenIDImport_err("Invalid data: invalid token value");
+	if ($entry[0] == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') PiratenIDImport_err("Invalid data: Token is hash of empty string");
+	
+	
+	if ($entry[1] !== 'ja' && $entry[1] !== 'nein') PiratenIDImport_err("Invalid data: mitgliedschaft-bund must be either 'ja' or 'nein'");
+	if (!in_array($entry[2], $LVs ,true)) {
+		PiratenIDImport_err("Invalid data: value for mitgliedschaft-land is not in whitelist");
+	}
+	if ($entry[6] !== 'ja' && $entry[6] !== 'nein' && $entry[6] !== '') PiratenIDImport_err("Invalid data: stimmberechtigt must be 'ja', 'nein' or empty string");
+}
\ No newline at end of file