Skip to content

Commit

Permalink
Mirror everything
Browse files Browse the repository at this point in the history 
Mirror all those repositories within which the user is a member. Update
the interfaces so that repository providers can return sets of groups.

Fix #2
  • Loading branch information
@io7m
io7m committed Feb 22, 2017
1 parent 23727db commit 28c5799
Show file tree
Hide file tree
Showing 8 changed files with 98 additions and 36 deletions.
9 changes: 9 additions & 0 deletions io7m-gtyrell-core/src/main/java/com/io7m/gtyrell/core/GTRepositoryGroupNameType.java
View file
Expand Up @@ -16,6 +16,7 @@

package com.io7m.gtyrell.core;

import com.io7m.jnull.NullCheck;
import org.immutables.value.Value;

import java.util.regex.Matcher;
Expand All @@ -27,6 +28,7 @@
@GTImmutableStyleType
@Value.Immutable
public interface GTRepositoryGroupNameType
extends Comparable<GTRepositoryGroupNameType>
{
/**
* @return The name
Expand Down Expand Up @@ -56,4 +58,11 @@ default void checkPreconditions()
throw new IllegalArgumentException(sb.toString());
}
}

@Override
default int compareTo(
final GTRepositoryGroupNameType o)
{
return this.text().compareTo(NullCheck.notNull(o, "Other").text());
}
}
4 changes: 2 additions & 2 deletions io7m-gtyrell-core/src/main/java/com/io7m/gtyrell/core/GTRepositoryGroupType.java
View file
Expand Up @@ -16,7 +16,7 @@

package com.io7m.gtyrell.core;

import javaslang.collection.Map;
import javaslang.collection.SortedMap;
import org.immutables.value.Value;

/**
Expand All @@ -39,5 +39,5 @@ public interface GTRepositoryGroupType
*/

@Value.Parameter
Map<GTRepositoryName, GTRepositoryType> repositories();
SortedMap<GTRepositoryName, GTRepositoryType> repositories();
}
10 changes: 9 additions & 1 deletion io7m-gtyrell-core/src/main/java/com/io7m/gtyrell/core/GTRepositoryNameType.java
View file
Expand Up @@ -16,6 +16,7 @@

package com.io7m.gtyrell.core;

import com.io7m.jnull.NullCheck;
import org.immutables.value.Value;

import java.util.regex.Matcher;
Expand All @@ -26,7 +27,7 @@

@GTImmutableStyleType
@Value.Immutable
public interface GTRepositoryNameType
public interface GTRepositoryNameType extends Comparable<GTRepositoryNameType>
{
/**
* @return The name
Expand Down Expand Up @@ -56,4 +57,11 @@ default void checkPreconditions()
throw new IllegalArgumentException(sb.toString());
}
}

@Override
default int compareTo(
final GTRepositoryNameType o)
{
return this.text().compareTo(NullCheck.notNull(o, "Other").text());
}
}
7 changes: 5 additions & 2 deletions io7m-gtyrell-core/src/main/java/com/io7m/gtyrell/core/GTRepositorySourceType.java
View file
Expand Up @@ -16,8 +16,11 @@

package com.io7m.gtyrell.core;

import javaslang.collection.SortedMap;

import java.io.IOException;


/**
* The type of functions that can fetch named groups of repositories.
*/
Expand All @@ -27,12 +30,12 @@ public interface GTRepositorySourceType
/**
* @param in_git A git executable
*
* @return A repository group
* @return A set of repository groups
*
* @throws IOException On I/O errors
*/

GTRepositoryGroupType get(
SortedMap<GTRepositoryGroupName, GTRepositoryGroupType> get(
GTGitExecutableType in_git)
throws IOException;
}
58 changes: 37 additions & 21 deletions io7m-gtyrell-github/src/main/java/com/io7m/gtyrell/github/GTGithubRepositories.java
View file
Expand Up @@ -24,8 +24,9 @@
import com.io7m.gtyrell.core.GTRepositorySourceType;
import com.io7m.gtyrell.core.GTRepositoryType;
import com.io7m.jnull.NullCheck;
import javaslang.collection.HashMap;
import javaslang.collection.Map;
import javaslang.Tuple;
import javaslang.collection.SortedMap;
import javaslang.collection.TreeMap;
import org.kohsuke.github.GHMyself;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GitHub;
Expand Down Expand Up @@ -85,7 +86,7 @@ public static GTRepositorySourceType newSource(
}

@Override
public GTRepositoryGroupType get(
public SortedMap<GTRepositoryGroupName, GTRepositoryGroupType> get(
final GTGitExecutableType in_git)
throws IOException
{
Expand All @@ -95,33 +96,48 @@ public GTRepositoryGroupType get(
final GitHubBuilder ghb = GitHubBuilder.fromProperties(this.props);
final GitHub gh = ghb.build();
final GHMyself me = gh.getMyself();
final String user = NullCheck.notNull(me.getLogin());
final GTRepositoryGroupName group = GTRepositoryGroupName.of(user);

Map<GTRepositoryName, GTRepositoryType> repositories = HashMap.empty();
SortedMap<GTRepositoryGroupName, SortedMap<GTRepositoryName, GTRepositoryType>> groups =
TreeMap.empty();

final PagedIterable<GHRepository> rs =
me.listRepositories(100, GHMyself.RepositoryListFilter.OWNER);
me.listRepositories(100, GHMyself.RepositoryListFilter.ALL);
final PagedIterator<GHRepository> rsi = rs.iterator();

while (rsi.hasNext()) {
final GHRepository r = rsi.next();

LOG.debug("repository: {} {}", r.getName(), r.getGitTransportUrl());

final GTRepositoryName name = GTRepositoryName.of(r.getName());
final URI clone_url = new URI(r.gitHttpTransportUrl());

repositories = repositories.put(
name,
LOG.debug(
"repository: {}/{} {}",
r.getOwnerName(),
r.getName(),
r.getGitTransportUrl());

final GTRepositoryGroupName group =
GTRepositoryGroupName.of(r.getOwnerName());
final GTRepositoryName name =
GTRepositoryName.of(r.getName());
final URI clone_url =
new URI(r.gitHttpTransportUrl());

SortedMap<GTRepositoryName, GTRepositoryType> repositories;
if (groups.containsKey(group)) {
repositories = groups.get(group).get();
} else {
repositories = TreeMap.empty();
}

final GTRepositoryType repository =
new GTGithubRepository(
in_git,
this.username,
this.password,
group,
name,
clone_url));
in_git, this.username, this.password, group, name, clone_url);

repositories = repositories.put(name, repository);
groups = groups.put(group, repositories);
}

return GTRepositoryGroup.of(group, repositories);
return groups.map(
(group_name, repositories) ->
Tuple.of(group_name, GTRepositoryGroup.of(group_name, repositories)));
} catch (final URISyntaxException e) {
throw new IOException(e);
}
Expand Down
29 changes: 20 additions & 9 deletions io7m-gtyrell-server/src/main/java/com/io7m/gtyrell/server/GTServer.java
View file
Expand Up @@ -22,8 +22,10 @@
import com.io7m.gtyrell.core.GTRepositorySourceType;
import com.io7m.gtyrell.core.GTRepositoryType;
import com.io7m.jnull.NullCheck;
import javaslang.Tuple2;
import javaslang.collection.List;
import javaslang.collection.Map;
import javaslang.collection.SortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -144,16 +146,21 @@ private void runOnce()
final GTRepositorySourceType p =
NullCheck.notNull(producers.get(index));

LOG.debug("retrieving repository group");
final GTRepositoryGroupType g;
LOG.debug("retrieving repository groups");
final SortedMap<GTRepositoryGroupName, GTRepositoryGroupType> groups;

try {
g = p.get(this.config.git());
} catch (final IOException e) {
LOG.error("error syncing group: ", e);
continue;
groups = p.get(this.config.git());
for (final Tuple2<GTRepositoryGroupName, GTRepositoryGroupType> group : groups) {
try {
this.syncGroup(group._2);
} catch (final Exception e) {
LOG.error("error syncing group: {}: ", group._1.text(), e);
}
}
} catch (final Exception e) {
LOG.error("error retrieving repository groups: ", e);
}

this.syncGroup(g);
}

LOG.debug("sync completed, pausing");
Expand All @@ -178,7 +185,11 @@ private void syncGroup(final GTRepositoryGroupType g)
try {
final File output =
makeRepositoryName(this.config.directory(), group, name);
repos.update(output);
if (!this.config.dryRun()) {
repos.update(output);
} else {
LOG.debug("not syncing due to dry run");
}
} catch (final IOException e) {
LOG.error("error syncing {}: ", repos, e);
}
Expand Down
11 changes: 11 additions & 0 deletions io7m-gtyrell-server/src/main/java/com/io7m/gtyrell/server/GTServerConfigurationType.java
View file
Expand Up @@ -61,4 +61,15 @@ public interface GTServerConfigurationType

@Value.Parameter
Duration pauseDuration();

/**
* @return {@code true} iff repositories should not actually be cloned
*/

@Value.Parameter
@Value.Default
default boolean dryRun()
{
return false;
}
}
6 changes: 5 additions & 1 deletion io7m-gtyrell-server/src/main/java/com/io7m/gtyrell/server/GTServerConfigurations.java
View file
Expand Up @@ -75,7 +75,11 @@ public static GTServerConfiguration fromProperties(
sources = sources.append(source);
}

return GTServerConfiguration.of(root, sources, git, pause);
final boolean dry_run =
JProperties.getBooleanOptional(
p, "com.io7m.gtyrell.server.dry_run", false);

return GTServerConfiguration.of(root, sources, git, pause, dry_run);
}

private static GTRepositorySourceType parseSource(
Expand Down

0 comments on commit 28c5799

Please sign in to comment.