Merge requests, environments, deployments & commits

The changes are all over the place, a little bit embarrasing, but no
point in cleaning this up now.

Author: squiddy

.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+charset = utf-8
+
+[*.py]
+indent_size = 4
+
+[src/Makefile]
+indent_style = tab

gitlab_to_sqlite/cli.py

@@ -68,6 +68,44 @@ def projects(db_path, project, auth):
     token, host = load_config(auth)
     project = utils.fetch_project(project, token, host)
     utils.save_project(db, project)
+    utils.ensure_db_shape(db)
+
+
+@cli.command(name="merge-requests")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("project", required=True)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+@click.option(
+    "--full",
+    is_flag=True,
+)
+def merge_requests(db_path, project, auth, full):
+    "Save merge requests"
+    db = sqlite_utils.Database(db_path)
+    token, host = load_config(auth)
+
+    new = 0
+    for merge_request in utils.fetch_merge_requests(
+        project,
+        token,
+        host,
+        None if full else utils.get_latest_merge_request_time(db, project),
+    ):
+        utils.save_merge_request(db, merge_request)
+        new += 1
+
+    utils.ensure_db_shape(db)
+    click.echo(f"Saved/updated {new} merge requests")
 
 
 @cli.command(name="pipelines")
@@ -84,20 +122,148 @@ def projects(db_path, project, auth):
     default="auth.json",
     help="Path to auth.json token file",
 )
-def pipelines(db_path, project, auth):
+@click.option(
+    "--full",
+    is_flag=True,
+)
+def pipelines(db_path, project, auth, full):
     "Save pipelines"
     db = sqlite_utils.Database(db_path)
     token, host = load_config(auth)
-    latest = utils.get_latest_pipeline_time(db, project)
 
     new = 0
-    for pipeline in utils.fetch_pipelines(project, token, host, latest):
-        utils.save_pipeline(db, pipeline)
+    for pipeline in utils.fetch_pipelines(
+        project,
+        token,
+        host,
+        None if full else utils.get_latest_pipeline_time(db, project),
+    ):
+        utils.save_pipeline(db, pipeline, host)
         new += 1
 
+    utils.ensure_db_shape(db)
     click.echo(f"Saved/updated {new} pipelines")
 
 
+@cli.command(name="environments")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("project", required=True)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def environments(db_path, project, auth):
+    db = sqlite_utils.Database(db_path)
+    token, host = load_config(auth)
+
+    new = 0
+    for environment in utils.fetch_environments(
+        project,
+        token,
+        host,
+    ):
+        utils.save_environment(db, environment)
+        new += 1
+
+    utils.ensure_db_shape(db)
+    click.echo(f"Saved/updated {new} environments")
+
+
+@cli.command(name="deployments")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("project", required=True)
+@click.argument("environment", required=True)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def deployments(db_path, project, environment, auth):
+    db = sqlite_utils.Database(db_path)
+    token, host = load_config(auth)
+
+    if (
+        "deployments" in db.table_names()
+        and "projects" in db.table_names()
+        and "environments" in db.table_names()
+    ):
+        r = db.query(
+            """
+        SELECT
+            max(d.updated_at) AS last_update
+        FROM
+            deployments d
+            JOIN projects p ON d.project_id = p.id
+            JOIN environments e ON d.environment_id = e.id
+        WHERE
+            p.full_path = ?
+            AND e.name = ?
+        """,
+            [project, environment],
+        )
+        last_update = next(r)["last_update"]
+    else:
+        last_update = None
+
+    new = 0
+    for deployment in utils.fetch_deployments(
+        project,
+        environment,
+        token,
+        host,
+        last_update,
+    ):
+        if utils.save_deployment(db, deployment) is not False:
+            new += 1
+
+    utils.ensure_db_shape(db)
+    click.echo(f"Saved/updated {new} deployments")
+
+
+@cli.command(name="commits")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("project", required=True)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def commits(db_path, project, auth):
+    db = sqlite_utils.Database(db_path)
+    token, host = load_config(auth)
+
+    new = 0
+    for commit in utils.fetch_commits(
+        project,
+        token,
+        host,
+    ):
+        utils.save_commit(db, commit)
+        new += 1
+
+    utils.ensure_db_shape(db)
+    click.echo(f"Saved/updated {new} commits")
+
+
 def load_config(auth):
     try:
         data = json.load(open(auth))