Create "import since" feature

When your Gatlab repo is old and hudge, may be you should not import all
your repo. Commits messages may respect the correct layout and do not
refer any mantis ticket number.
This feature can also avoid erors during very large imports.
When your report is very hudge, reimport all commits could be
very timeconsuming, and the result is so huge that php engine can raise
a memory error.

fix "deprecated dynamic parrameter" creation.

Author: Zic0-91

Source/MantisSourceGitBasePlugin.class.php

@@ -46,6 +46,7 @@ abstract class MantisSourceGitBasePlugin extends MantisSourcePlugin
 	 * Error constants
 	 */
 	const ERROR_INVALID_BRANCH = 'invalid_branch';
+	const ERROR_INVALID_DATE = 'invalid_date';
 
 	/**
 	 * Define plugin's Error strings
@@ -54,6 +55,7 @@ abstract class MantisSourceGitBasePlugin extends MantisSourcePlugin
 	public function errors() {
 		$t_errors_list = array(
 			self::ERROR_INVALID_BRANCH,
+			self::ERROR_INVALID_DATE,
 		);
 
 		foreach( $t_errors_list as $t_error ) {
@@ -132,6 +134,21 @@ protected function get_default_primary_branches() {
 		return $t_value;
 	}
 
+  /**
+	 * Validates a date
+	 * Triggers an ERROR_INVALID_DATE if date is not valid
+	 * @return void
+	 */
+	protected function validate_date($p_date) {
+		if (empty($p_date)) {
+			return;
+		}
+    if (! (bool)strtotime($p_date)) {
+			error_parameters( $p_date );
+			plugin_error( self::ERROR_INVALID_DATE );
+	}
+}
+
 	/**
 	 * Output form elements for configuration options.
 	 */

Source/lang/strings_english.txt

@@ -138,7 +138,9 @@ $s_plugin_Source_changeset_removed = 'Changeset removed';
 
 $s_plugin_Source_ensure_delete = 'Do you really want to delete the "%s" repository and all of its history?';
 $s_plugin_Source_ensure_import_full = 'This will import to a new copy of your repository, and then destroy the old repository data, and may require use of shell access.  Do you wish to continue?';
+$s_plugin_Source_ensure_import__full_since = 'This will import to a new copy of your repository since "%s", and then destroy the old repository data, and may require use of shell access. Do you wish to continue?';
 $s_plugin_Source_ensure_import_latest = 'This will attempt to import recent data for your repository, and may require use of shell access.  Do you wish to continue?';
+$s_plugin_Source_ensure_import_latest_since = 'This will attempt to import since "%s" for yout repository,  and may require use of shell access.  Do you wish to continue?';
 $s_plugin_Source_import_results = 'Import Results';
 $s_plugin_Source_import_stats = 'Imported %s changesets, %s files, and %s bug references.';
 $s_plugin_Source_import_repo_error = 'Import process produced an error.';
@@ -150,6 +152,7 @@ $s_plugin_Source_invalid_changeset = 'Changeset information could not be loaded'
 
 $s_plugin_Source_import_latest_failed = 'Repository latest data importing failed.';
 $s_plugin_Source_import_full_failed = 'Full repository data importing failed.';
+$s_plugin_Source_import_since_failed = 'Repository "since date" importing failed.';
 
 $s_plugin_Source_changeset_column_title = 'C';
 

Source/lang/strings_french.txt

@@ -70,6 +70,7 @@ $s_plugin_Source_back_repo = 'Retour au dépôt';
 $s_plugin_Source_back_changeset = 'Retour au jeu de changements';
 $s_plugin_Source_import_full = 'Tout importer';
 $s_plugin_Source_import_latest = 'Importer les dernières données';
+$s_plugin_Source_import_since = 'Importer depuis le...';
 $s_plugin_Source_related_changesets = 'Jeux de changements liés';
 $s_plugin_Source_affected_issues = 'Demandes affectées';
 $s_plugin_Source_attach_to_issue = 'Attacher des demandes :';
@@ -134,7 +135,9 @@ $s_plugin_Source_changeset_removed = 'Jeu de changement détaché';
 
 $s_plugin_Source_ensure_delete = 'Voulez-vous vraiment supprimer le dépôt "%s" et tout son historique ?';
 $s_plugin_Source_ensure_import_full = 'Ceci va importer une nouvelle copie de votre dépôt, puis détruira les vieilles données de dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
+$s_plugin_Source_ensure_import_full_since = 'Ceci va importer une nouvelle copie de votre dépôt depuis le "%s", puis détruira les vieilles données de dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
 $s_plugin_Source_ensure_import_latest = 'Ceci va tenter d\'importer les données récentes dans votre dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
+$s_plugin_Source_ensure_import_latest_since = 'Cevi va tenter d\'importer les données depuis le "%s" dans votre dépôt et pourrait avoir besoin d\'un accès au shell. Voulez-vous continuer ?';
 $s_plugin_Source_import_results = 'Résultats de l\'import';
 $s_plugin_Source_import_stats = '%s jeux de changements importés, %s fichiers, et %s références de demande.';
 $s_plugin_Source_import_repo_error = 'Le processus d\'import a rencontré une erreur.';

Source/pages/repo_import_full.php

@@ -6,12 +6,18 @@
 form_security_validate( 'plugin_Source_repo_import_full' );
 access_ensure_global_level( plugin_config_get( 'manage_threshold' ) );
 
-$f_repo_id = gpc_get_string( 'id' );
+$f_repo_id = gpc_get_int( 'id' );
 
 $t_repo = SourceRepo::load( $f_repo_id );
 $t_vcs = SourceVCS::repo( $t_repo );
 
-helper_ensure_confirmed( plugin_lang_get( 'ensure_import_full' ), plugin_lang_get( 'import_full' ) );
+if (empty( $t_repo->info['hub_oldest_commit_date'])){
+	helper_ensure_confirmed( plugin_lang_get( 'ensure_import_full' ), plugin_lang_get( 'import_full' ) );
+}else{
+	$t_date_since = $t_repo->info['hub_oldest_commit_date'];
+	helper_ensure_confirmed( sprintf( plugin_lang_get( 'ensure_import_full_since' ), $t_date_since), plugin_lang_get( 'import_full' ) );
+}
+
 helper_begin_long_process();
 
 layout_page_header( plugin_lang_get( 'title' ) );

Source/pages/repo_import_latest.php

@@ -7,17 +7,23 @@
 
 form_security_validate( 'plugin_Source_repo_import_latest' );
 access_ensure_global_level( plugin_config_get( 'manage_threshold' ) );
-helper_ensure_confirmed( plugin_lang_get( 'ensure_import_latest' ), plugin_lang_get( 'import_latest' ) );
 
-$f_repo_id = strtolower( gpc_get_string( 'id' ) );
+$f_repo_id = gpc_get_int( 'id' );
 
 $t_repo_id = (int) $f_repo_id;
 $t_repos = array( SourceRepo::load( $t_repo_id ) );
-
 $t_repo = array_shift( $t_repos );
 $t_vcs = SourceVCS::repo( $t_repo );
 
-$t_repo->pre_stats = $t_repo->stats();
+$t_pre_stats = $t_repo->stats();
+
+if ( empty( $t_repo->info['hub_oldest_commit_date'] ) ) {
+	helper_ensure_confirmed( plugin_lang_get( 'ensure_import_latest' ), plugin_lang_get( 'import_latest' ) );
+}else{
+	$t_date_since = $t_repo->info['hub_oldest_commit_date'];
+	helper_ensure_confirmed( sprintf(plugin_lang_get( 'ensure_import_latest_since' ), $t_date_since), plugin_lang_get( 'import_latest' ) );
+}
+
 
 layout_page_header( plugin_lang_get( 'title' ) );
 layout_page_begin();
@@ -39,15 +45,15 @@
 
 <?php
 # keep checking for more changesets to import
-$t_repo->import_error = false;
+$t_import_error = false;
 while( true ) {
 
 	# import the next batch of changesets
 	$t_changesets = $t_vcs->import_latest( $t_repo );
 
 	# check for errors
 	if ( !is_array( $t_changesets ) ) {
-		$t_repo->import_error = true;
+		$t_import_error = true;
 		break;
 	}
 
@@ -59,21 +65,21 @@
 	Source_Process_Changesets( $t_changesets );
 }
 
-$t_repo->post_stats = $t_repo->stats();
+$t_post_stats = $t_repo->stats();
 ?>
 
 <tr>
 	<td class="category"><?php echo string_display_line( $t_repo->name ) ?></td>
 <td>
 <?php
-if ( $t_repo->import_error ) {
+if ( $t_import_error) {
 	echo plugin_lang_get( 'import_latest_failed' ), '<br/>';
 }
 
-$t_stats = $t_repo->post_stats;
-$t_stats['changesets'] -= $t_repo->pre_stats['changesets'];
-$t_stats['files'] -= $t_repo->pre_stats['files'];
-$t_stats['bugs'] -= $t_repo->pre_stats['bugs'];
+$t_stats = $t_post_stats;
+$t_stats['changesets'] -= $t_pre_stats['changesets'];
+$t_stats['files'] -= $t_pre_stats['files'];
+$t_stats['bugs'] -= $t_pre_stats['bugs'];
 
 echo sprintf( plugin_lang_get( 'import_stats' ), $t_stats['changesets'], $t_stats['files'], $t_stats['bugs'] );
 ?>

Source/pages/repo_manage_page.php

@@ -177,12 +177,12 @@ function convert_to_key_value( $p_array ) {
 					<form action="<?php echo plugin_page( 'repo_import_latest' ) . '&amp;id=' . $t_repo->id ?>" method="post" class="pull-right">
 						<?php echo form_security_field( 'plugin_Source_repo_import_latest' ) ?>
 						<input type="submit" class="btn btn-primary btn-white btn-sm btn-round " value="<?php echo plugin_lang_get( 'import_latest' ) ?>"/>
-					</form>
+					</form>	
 				</div>
 			</div>
 		</div>
 	</div>
-
+	
 	<div class="space-10"></div>
 
 

SourceGitlab/SourceGitlab.php

@@ -94,6 +94,7 @@ public function update_repo_form( $p_repo ) {
 		$t_hub_repoid = null;
 		$t_hub_reponame = null;
 		$t_hub_app_secret = null;
+		$t_hub_oldest_commit_date = null;
 
 		if ( isset( $p_repo->info['hub_root'] ) ) {
 			$t_hub_root = $p_repo->info['hub_root'];
@@ -112,6 +113,9 @@ public function update_repo_form( $p_repo ) {
 		} else {
 			$t_master_branch = $this->get_default_primary_branches();
 		}
+		if ( isset( $p_repo->info['hub_oldest_commit_date'] ) ) {
+			$t_hub_oldest_commit_date = $p_repo->info['hub_oldest_commit_date'];
+		}
 ?>
 <tr>
 	<th class="category">
@@ -182,6 +186,18 @@ public function update_repo_form( $p_repo ) {
 		/>
 	</td>
 </tr>
+<tr>
+	<th class="category">
+		<label for="hub_oldest_commit_date">
+			<?php echo plugin_lang_get( 'hub_oldest_commit_date' ) ?>
+		</label>
+	</th>
+	<td>
+		  <input type="date" id="hub_oldest_commit_date" name="hub_oldest_commit_date" 
+		           value="<?php echo string_attribute( $t_hub_oldest_commit_date ) ?>"
+			/>
+	</td>
+</tr>
 <?php
 	}
 
@@ -240,6 +256,13 @@ public function update_repo( $p_repo ) {
 		$p_repo->info['hub_repoid'] = $f_hub_repoid;
 		$p_repo->info['master_branch'] = $f_master_branch;
 
+		
+		# Check date format
+		$t_hub_oldest_commit_date = gpc_get_string( 'hub_oldest_commit_date' );
+		$this->validate_date($t_hub_oldest_commit_date);
+		
+		$p_repo->info['hub_oldest_commit_date'] = $t_hub_oldest_commit_date;
+
 		return $p_repo;
 	}
 
@@ -327,8 +350,10 @@ protected function build_gitlab_apis($p_repo, $t_branch) {
 	}
 
 
-	public function import_full( $p_repo ) {
-		echo '<pre>';
+	public function import_full( $p_repo) {
+		static $counter=0;
+		$counter = $counter + 1;
+		echo "<pre>Pass #$counter: \n";
 
 		$t_branch = $p_repo->info['master_branch'];
 		if ( is_blank( $t_branch ) ) {
@@ -382,17 +407,18 @@ public function import_full( $p_repo ) {
 				WHERE repo_id=" . db_param() . ' AND branch=' . db_param() .
 				' ORDER BY timestamp';
 			$t_result = db_query( $t_query, array( $p_repo->id, $t_branch->name ), 1 );
-
+			echo "Process branch '$t_branch->name':\n";
 			$t_commits = array( $t_branch->commit->id );
 			if ( db_num_rows( $t_result ) > 0 ) {
 				$t_parent = db_result( $t_result );
 				echo "Oldest '$t_branch->name' branch parent: '$t_parent'\n";
 
 				if ( !empty( $t_parent ) ) {
 					$t_commits[] = $t_parent;
-					echo "Parents not empty";
+					echo "Parents not empty - ";
+				} else {
+					echo "Parents  empty";
 				}
-				echo "Parents  empty";
 			}
 
 			$t_changesets = array_merge( $t_changesets, $this->import_commits( $p_repo, $t_commits, $t_branch->name ) );
@@ -449,37 +475,43 @@ public function import_commits( $p_repo, $p_commit_ids, $p_branch='' ) {
 
 	private function json_commit_changeset( $p_repo, $p_json, $p_branch='' ) {
 		echo "processing $p_json->id ... ";
-		if ( !SourceChangeset::exists( $p_repo->id, $p_json->id ) ) {
-			# Message will be replaced by title in gitlab version earlier than 7.2
-			$t_message = ( !property_exists( $p_json, 'message' ) )
-				? $p_json->title
-				: $p_json->message;
-			$t_changeset = new SourceChangeset(
-				$p_repo->id,
-				$p_json->id,
-				$p_branch,
-				$p_json->created_at,
-				$p_json->author_name,
-				$t_message
-			);
-
-			$t_parents = array();
-			foreach( $p_json->parent_ids as $t_parent ) {
-				$t_parents[] = $t_parent;
-			}
-			if( $t_parents ) {
-				$t_changeset->parent = $t_parents[0];
-			}
-
-			$t_changeset->author_email = $p_json->author_email;
-			$t_changeset->save();
-
-			echo "saved.\n";
-			return array( $t_changeset, $t_parents );
-		} else {
+		if ( SourceChangeset::exists( $p_repo->id, $p_json->id ) ) {
 			echo "already exists.\n";
 			return array( null, array() );
 		}
+
+		$t_oldest_commit_date = $p_repo->info['hub_oldest_commit_date']; 
+		if ( !empty( $t_oldest_commit_date ) && new DateTime( $p_json->created_at ) < new DateTime( $t_oldest_commit_date ) ) {
+			echo "commit before the oldest_commit_date.\n";
+			return array( null, array() );
+		}
+		
+		# Message will be replaced by title in gitlab version earlier than 7.2
+		$t_message = ( !property_exists( $p_json, 'message' ) )
+			? $p_json->title
+			: $p_json->message;
+		$t_changeset = new SourceChangeset(
+			$p_repo->id,
+			$p_json->id,
+			$p_branch,
+			$p_json->created_at,
+			$p_json->author_name,
+			$t_message
+		);
+
+		$t_parents = array();
+		foreach( $p_json->parent_ids as $t_parent ) {
+			$t_parents[] = $t_parent;
+		}
+		if( $t_parents ) {
+			$t_changeset->parent = $t_parents[0];
+		}
+
+		$t_changeset->author_email = $p_json->author_email;
+		$t_changeset->save();
+
+		echo "saved.\n";
+		return array( $t_changeset, $t_parents );
 	}
 
 	public static function url_post( $p_url, $p_post_data ) {

SourceGitlab/lang/strings_english.txt

@@ -14,6 +14,7 @@ $s_plugin_SourceGitlab_hub_repoid = 'GitLab Repository ID<br /><span class="smal
 $s_plugin_SourceGitlab_hub_reponame = 'GitLab Repository Name<br /><span class="small">Name &amp; Namespace with no leading or trailing slashes (namespace/project)</span>';
 $s_plugin_SourceGitlab_hub_app_secret = 'GitLab API Key<br /><span class="small">api key for mantis user (continuous user)</span>';
 $s_plugin_SourceGitlab_master_branch = 'Allowed Branches<br/><span class="small">(comma-separated list, regular expression with /.../  or *)</span>';
+$s_plugin_SourceGitlab_hub_oldest_commit_date = 'Import since (optional)<br/><span class="small">Oldest commit will be ignored, leave empty to import all commits</span>';
 $s_plugin_SourceGitlab_import_limit = 'Import Limit<br/><span class="small">at most this many commits per import action (0: unlimited)</span>';
 
 $s_plugin_SourceGitlab_error_api = 'GitLab API error (%1$s)';