Skip to content

Commit

Permalink
IGNITE-9409 Use current version, modern URL for Ignite download - Fixes
Browse files Browse the repository at this point in the history 
#6599.

Signed-off-by: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
  • Loading branch information
@alamar
alamar committed Jun 13, 2019
1 parent 8a6c0d4 commit ebb3659
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 147 deletions.
25 changes: 25 additions & 0 deletions modules/yarn/pom.xml
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -99,6 +99,31 @@


<build> <build>
<plugins> <plugins>
<plugin>
<!-- Copy ignite.properties from core module since it is not available as dependency. -->
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>copy-resources</id>
<phase>validate</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources</outputDirectory>
<resources>
<resource>
<directory>../core/src/main/resources</directory>
<includes>
<include>ignite.properties</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<artifactId>maven-assembly-plugin</artifactId> <artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version> <version>2.4.1</version>
Expand Down
217 changes: 75 additions & 142 deletions modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -21,33 +21,29 @@
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.nio.channels.Channels; import java.nio.channels.Channels;
import java.util.ArrayList; import java.util.Properties;
import java.util.Collections; import java.util.logging.Logger;
import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.ignite.yarn.utils.IgniteYarnUtils; import org.apache.ignite.yarn.utils.IgniteYarnUtils;


/** /**
* Downloads and stores Ignite. * Downloads and stores Ignite release.
*/ */
public class IgniteProvider { public class IgniteProvider {
/** */ /** */
public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php"; public static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName());


/** */ /** */
private ClusterProperties props; public static final String DOWNLOAD_LINK = "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/";

/** */
private String latestVersion = null;


/** */ /** */
private boolean hdfs = false; private ClusterProperties props;


/** */ /** */
private FileSystem fs; private FileSystem fs;
Expand All @@ -62,93 +58,67 @@ public IgniteProvider(ClusterProperties props, FileSystem fs) {
} }


/** /**
* @return Latest ignite version. * @return Path to current Ignite release.
*/ */
public Path getIgnite() throws Exception { public Path getIgnite() throws Exception {
File folder = checkDownloadFolder(); File folder = checkDownloadFolder();


if (latestVersion == null) { Properties verProps = new Properties();
List<String> localFiles = findIgnites(folder);
List<String> hdfsFiles = findIgnites(fs, props.igniteReleasesDir());


String localLatestVersion = findLatestVersion(localFiles); try (InputStream is = IgniteProvider.class.getClassLoader().getResourceAsStream("ignite.properties")) {
String hdfsLatestVersion = findLatestVersion(hdfsFiles); verProps.load(is);
}


if (localLatestVersion != null && hdfsLatestVersion != null) { String curVer = verProps.getProperty("ignite.version");
if (VersionComparator.INSTANCE.compare(hdfsLatestVersion, localLatestVersion) >= 0) {
latestVersion = hdfsLatestVersion;


hdfs = true; if (curVer == null || curVer.isEmpty())
} throw new IllegalStateException("Failed to determine Ignite version");
}
else if (localLatestVersion != null)
latestVersion = localLatestVersion;
else if (hdfsLatestVersion != null) {
latestVersion = hdfsLatestVersion;


hdfs = true; log.info("Searching for Ignite release " + curVer);
}
}


String newVersion = updateIgnite(latestVersion); File release = findIgnite(folder, curVer);


if (latestVersion != null && newVersion.equals(latestVersion)) { if (release == null) {
if (hdfs) Path releaseOnHdfs = findIgnite(fs, props.igniteReleasesDir(), curVer);
return new Path(formatPath(props.igniteReleasesDir(), latestVersion));
else if (releaseOnHdfs != null)
return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion), return releaseOnHdfs;
formatPath(props.igniteReleasesDir(), latestVersion));
}
else {
latestVersion = newVersion;


return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion), release = updateIgnite(curVer);
formatPath(props.igniteReleasesDir(), latestVersion));
} }

return IgniteYarnUtils.copyLocalToHdfs(fs, release.getAbsolutePath(),
props.igniteReleasesDir() + File.separator + release.getName());
} }


/** /**
* @param folder Folder. * @param folder Folder.
* @param curVer Ignite version.
* @return Ignite archives. * @return Ignite archives.
*/ */
private List<String> findIgnites(File folder) { private File findIgnite(File folder, String curVer) {
String[] files = folder.list(); String[] files = folder.list();


List<String> ignites = new ArrayList<>();

if (files != null) { if (files != null) {
for (String fileName : files) { for (String fileName : files) {
if (fileName.contains("gridgain-professional-") && fileName.endsWith(".zip")) if (fileName.equals(igniteRelease(curVer))) {
ignites.add(fileName); log.info("Found local release at " + folder.getAbsolutePath());
}
}


return ignites; return new File(folder, fileName);
} }

}
/**
* @param files Files.
* @return latest ignite version.
*/
private String findLatestVersion(List<String> files) {
String latestVersion = null;

if (!files.isEmpty()) {
if (files.size() == 1)
latestVersion = parseVersion(files.get(0));
else
latestVersion = parseVersion(Collections.max(files, VersionComparator.INSTANCE));
} }


return latestVersion; return null;
} }


/** /**
* @param fs File system, * @param fs File system,
* @param folder Folder. * @param folder Folder.
* @param curVer Ignite version.
* @return Ignite archives. * @return Ignite archives.
*/ */
private List<String> findIgnites(FileSystem fs, String folder) { private Path findIgnite(FileSystem fs, String folder, String curVer) {
FileStatus[] fileStatuses = null; FileStatus[] fileStatuses = null;


try { try {
Expand All @@ -158,86 +128,89 @@ private List<String> findIgnites(FileSystem fs, String folder) {
// Ignore. Folder doesn't exist. // Ignore. Folder doesn't exist.
} }
catch (Exception e) { catch (Exception e) {
throw new RuntimeException("Couldnt get list files from hdfs.", e); throw new RuntimeException("Couldn't get list files from hdfs.", e);
} }


List<String> ignites = new ArrayList<>();

if (fileStatuses != null) { if (fileStatuses != null) {
for (FileStatus file : fileStatuses) { for (FileStatus file : fileStatuses) {
String fileName = file.getPath().getName(); String fileName = file.getPath().getName();


if (fileName.contains("gridgain-professional-") && fileName.endsWith(".zip")) if (fileName.equals(igniteRelease(curVer))) {
ignites.add(fileName); log.info("Found HDFS release at " + file.getPath());

return file.getPath();
}
} }
} }


return ignites; return null;
} }


/** /**
* @param version Ignite version. * @param igniteUrl Download link.
* @return Ignite. * @return Ignite.
*/ */
public Path getIgnite(String version) throws Exception { public Path getIgnite(String igniteUrl) throws Exception {
checkDownloadFolder(); checkDownloadFolder();


// Download ignite. // Download ignite.
String fileName = downloadIgnite(version); String fileName = downloadIgnite(igniteUrl);


Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName); Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName);


log.info("Using specified release at " + igniteUrl);

if (!fs.exists(dst)) if (!fs.exists(dst))
fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst); fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst);


return dst; return dst;
} }


/** /**
* @param folder folder * @return File name.
* @param version version
* @return Path
*/ */
private static String formatPath(String folder, String version) { private static String igniteRelease(String version) {
return folder + File.separator + "gridgain-professional-" + version + ".zip"; return "apache-ignite-" + version + "-bin.zip";
} }


/** /**
* @param currentVersion The current latest version. * @param curVer Ignite version.
* @return Current version if the current version is latest; new ignite version otherwise. * @return Current version if the current version is latest; new ignite version otherwise.
*/ */
private String updateIgnite(String currentVersion) { private File updateIgnite(String curVer) {
try { try {
URL url; // Such as https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/ignite/2.7.0/apache-ignite-2.7.0-bin.zip

URL url = new URL(DOWNLOAD_LINK + curVer + "/" + igniteRelease(curVer));
if (currentVersion == null)
url = new URL(DOWNLOAD_LINK);
else
url = new URL(DOWNLOAD_LINK + "?version=" + currentVersion);


HttpURLConnection conn = (HttpURLConnection)url.openConnection(); HttpURLConnection conn = (HttpURLConnection)url.openConnection();


int code = conn.getResponseCode(); int code = conn.getResponseCode();


if (code == 200) { String redirectUrl = conn.getURL().toString();
String redirectUrl = conn.getURL().toString();

checkDownloadFolder();


FileOutputStream outFile = new FileOutputStream(props.igniteLocalWorkDir() + File.separator if (code == 301 || code == 302) {
+ fileName(redirectUrl)); redirectUrl = conn.getHeaderField("Location");

outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);


outFile.close(); conn.disconnect();


return parseVersion(redirectUrl); conn = (HttpURLConnection)new URL(redirectUrl).openConnection();
} }
else if (code == 304) else if (code != 200)
// This version is latest.
return currentVersion;
else
throw new RuntimeException("Got unexpected response code. Response code: " + code); throw new RuntimeException("Got unexpected response code. Response code: " + code);

checkDownloadFolder();

File ignite = new File(props.igniteLocalWorkDir(), fileName(redirectUrl));

FileOutputStream outFile = new FileOutputStream(ignite);

outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE);

outFile.close();

log.info("Found remote release at " + redirectUrl);

return ignite;
} }
catch (IOException e) { catch (IOException e) {
throw new RuntimeException("Failed update ignite.", e); throw new RuntimeException("Failed update ignite.", e);
Expand Down Expand Up @@ -295,16 +268,6 @@ private File checkDownloadFolder() {
return file; return file;
} }


/**
* @param url URL.
* @return Ignite version.
*/
private static String parseVersion(String url) {
String[] split = url.split("-");

return split[split.length - 1].replaceAll(".zip", "");
}

/** /**
* @param url URL. * @param url URL.
* @return File name. * @return File name.
Expand All @@ -314,34 +277,4 @@ private static String fileName(String url) {


return split[split.length - 1]; return split[split.length - 1];
} }

/**
* Ignite version comparator.
*/
public static final class VersionComparator implements Comparator<String> {
/** */
public static final VersionComparator INSTANCE = new VersionComparator();

/** */
private VersionComparator() {
// No-op.
}

/** {@inheritDoc} */
@Override public int compare(String f1, String f2) {
if (f1.equals(f2))
return 0;

String[] ver1 = parseVersion(f1).split("\\.");
String[] ver2 = parseVersion(f2).split("\\.");

if (Integer.valueOf(ver1[0]) >= Integer.valueOf(ver2[0])
&& Integer.valueOf(ver1[1]) >= Integer.valueOf(ver2[1])
&& Integer.valueOf(ver1[2]) >= Integer.valueOf(ver2[2]))

return 1;
else
return -1;
}
}
} }
12 changes: 7 additions & 5 deletions modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public static void main(String[] args) throws Exception {


String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);


if (tokRenewer == null || tokRenewer.length() == 0) if (tokRenewer == null || tokRenewer.isEmpty())
throw new IOException("Master Kerberos principal for the RM is not set."); throw new IOException("Master Kerberos principal for the RM is not set.");


log.info("Found RM principal: " + tokRenewer); log.info("Found RM principal: " + tokRenewer);
Expand Down Expand Up @@ -190,10 +190,12 @@ private static void checkArguments(String[] args) {
private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception { private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception {
IgniteProvider provider = new IgniteProvider(props, fileSystem); IgniteProvider provider = new IgniteProvider(props, fileSystem);


if (props.igniteUrl() == null) String igniteUrl = props.igniteUrl();
return provider.getIgnite();
else if (igniteUrl != null)
return provider.getIgnite(props.igniteUrl()); return provider.getIgnite(igniteUrl);

return provider.getIgnite();
} }


/** /**
Expand Down
Loading

0 comments on commit ebb3659

Please sign in to comment.