Permalink
Browse files

multithread download, download resume, used to download video from cs…

…s-tricks.com
  • Loading branch information...
feng
feng committed Mar 7, 2011
1 parent ebf83eb commit e4962c7e3ceaba20a3e8b2c722a81a57e0d4d6bc
@@ -0,0 +1,79 @@
+15-introduction-to-firebug
+39-how-to-use-sifr-3
+35-intro-to-jquery-2
+74-editable-css3-image-gallery
+59-embedding-audio
+28-using-wufoo-for-web-forms
+60-ajax-refreshing-rss-content
+43-how-to-use-css-sprites
+70-random-pet-peeves
+17-sliding-doors-button
+38-basics-tips-on-designing-for-the-iphone
+29-google-search-for-your-site
+91-the-wordpress-loop
+22-cutting-clipping-paths
+25-designing-for-wordpress-part-one
+67-jquery-part-3-image-title-plugin
+86-mamp
+33-ecommerce-product-page-makeover
+23-conditional-stylesheets
+93-css3-slideup-boxes
+13-converting-a-photoshop-mockup-part-two-episode-two
+31-introduction-to-google-analytics
+63-on-screencasting
+18-introduction-to-the-band-website-template
+76-jquery-tour
+85-best-practices-dynamic-content
+36-current-nav-highlighting-using-php-to-set-the-body-id
+65-advanced-uses-for-custom-fields-in-wordpress
+71-building-a-website-photoshop-mockup
+8-css-formatting
+64-building-a-photo-gallery
+4-forcing-scrollbars-eliminating-horizonatal-jumps
+66-table-styling-2-fixed-header-and-highlighting
+5-columns-of-equal-height-super-simple-two-column-layout
+40-how-z-index-works
+34-integrating-and-customizing-google-maps
+42-all-about-floats-screencast
+83-thoughts-on-seo
+61-basic-table-styling
+79-complete-jquery-animations
+81-ajaxing-a-wordpress-theme
+90-simple-textmate-tips
+14-converting-a-photoshop-mockup-part-two-episode-three
+41-wordpress-as-a-cms
+19-designing-a-unique-page-for-twitter-updates
+84-site-walkthrough
+77-styling-an-individual-article
+3-converting-a-photoshop-mockup-part-3-of-3
+73-wordpress-theme
+88-intro-to-compass-sass
+94-intro-to-pseudo-elements
+58-html-css-the-very-basics
+78-on-web-advertising
+87-moving-up-with-mamp
+89-organizing-photoshop
+37-status-fluid-and-menu-bar-apps
+92-drawing-table
+16-creating-the-photoshop-mockup
+20-introduction-to-jquery
+11-working-modularly-with-php
+68-think-geek-background-fade-technique
+9-starry-night-3d-background-with-the-parallax-effect
+69-first-ten-minutes-with-typekit
+30-creating-and-sending-html-email
+24-rounded-corners
+32-using-the-unit-png-fix
+72-building-a-website-2-of-3-htmlcss-conversion
+2-converting-a-photoshop-mockup-part-2-of-3
+75-how-not-to-design-a-checkout
+80-regarding-wheel-invention
+7-three-state-menu
+62-advanced-form-styling-functionality
+82-css-image-switcher
+12-converting-a-photoshop-mockup-part-two-episode-one
+10-fixed-width-fluid-width-elastic-width
+27-designing-for-wordpress-part-three
+21-walkthrough-of-contact-form
+26-designing-for-wordpress-part-two
+6-tools-of-the-trade
@@ -0,0 +1,43 @@
+package csstricks;
+
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class CountDownLatchTest {
+
+ private static class MyRunnable implements Runnable {
+
+ private CountDownLatch latch;
+
+ public MyRunnable(CountDownLatch latch) {
+ this.latch = latch;
+ }
+
+ @Override
+ public void run() {
+ try {
+ TimeUnit.MILLISECONDS.sleep(new Random().nextInt(1000));
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ latch.countDown();
+// latch.
+ System.out.println("over");
+ }
+
+ }
+
+ public static void main(String[] args) throws InterruptedException {
+
+ int SIZE = 3;
+ CountDownLatch latch = new CountDownLatch(SIZE);
+ for (int i = 0; i < SIZE; i++) {
+ new Thread(new MyRunnable(latch)).start();
+ }
+
+ latch.await();
+ System.out.println("all finished");
+
+ }
+}
@@ -0,0 +1,159 @@
+package csstricks;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.DecimalFormat;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+
+public class Download {
+ private static Logger logger = Logger.getLogger(Download.class);
+
+ public static void main(String[] args) throws IOException {
+ Download manager = new Download();
+ // String
+ manager.download(new URL(
+ "http://files/java/eclipse-jee-helios-SR1-linux-gtk.tar.gz"),
+ new File("/tmp/what.gif"));
+ }
+
+ public Set<String> getURLs(URL url) throws IOException {
+ InputStream stream = url.openConnection().getInputStream();
+ BufferedReader br = new BufferedReader(new InputStreamReader(stream));
+ String line = null;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ sb.append(line).append("\n");
+ }
+ // System.out.println(sb.toString());
+ Set<String> urls = new HashSet<String>();
+
+ Pattern p = Pattern.compile("\\shref=\"(.+?)\"");
+ Matcher matcher = p.matcher(sb);
+ while (matcher.find()) {
+ String u = matcher.group(1);
+ urls.add(u);
+ }
+ return urls;
+ }
+
+ public long getLength(URL url) throws IOException {
+ HttpURLConnection con = (HttpURLConnection) url.openConnection();
+ con.setRequestMethod("HEAD");
+ return Long.parseLong(con.getHeaderField("Content-Length"));
+ }
+
+ private static final DecimalFormat df = new DecimalFormat("##.##");
+
+ private String percent(long total, long finished) {
+ double p = (double) finished / total * 100;
+ return df.format(p);
+ }
+
+ private static String format(double d) {
+ return df.format(d);
+ }
+
+ public void download(URL url, File dest) throws IOException {
+ logger.info(url + "-->" + dest);
+ for (int i = 0; i < 30; ++i) {
+ try {
+ HttpURLConnection con = (HttpURLConnection) url
+ .openConnection();
+ long readed = 0; // bytes this time download
+ long totalLength = getLength(url); // total length of the
+ // resource
+ long downloaded = 0; // already downloaded before this try
+ double lastPercent = 0.0;
+ if (dest.exists()) {
+ downloaded = dest.length();
+ if (totalLength == downloaded) {
+ logger.info("already finished, exits");
+ return;
+ } else if (totalLength < downloaded) {
+ logger.error(downloaded + " > " + totalLength
+ + "; error!");
+ return;
+ }
+ con.addRequestProperty("range", "bytes=" + downloaded + "-"
+ + totalLength);
+ logger.info("resume download; " + downloaded + "/"
+ + totalLength + "; "
+ + percent(totalLength, downloaded));
+ lastPercent = (double) downloaded / totalLength;
+ }
+ con.connect();
+
+ Map<String, List<String>> headers = con.getHeaderFields();
+ for (Entry<String, List<String>> e : headers.entrySet()) {
+ logger.info(e.getKey() + ": " + e.getValue());
+ }
+
+ InputStream stream = con.getInputStream();
+ BufferedOutputStream bos = new BufferedOutputStream(
+ new FileOutputStream(dest, true));
+ long startTime = System.currentTimeMillis();
+ long lastReadPosition = readed;
+ long lastTime = startTime;
+
+ byte[] buffer = new byte[8912 * 2];
+ int read = -1;
+
+ while ((read = stream.read(buffer)) != -1) {
+ readed += read;
+ bos.write(buffer, 0, read);
+ double percent = (double) (readed + downloaded)
+ / totalLength;
+ if (percent - lastPercent > 0.01) {
+ long currentTime = System.currentTimeMillis();
+ String lastSpeed = format((double) (readed - lastReadPosition)
+ / (currentTime - lastTime) * 1000 / 1024)
+ + "k/s";
+
+ String speed = format((double) readed
+ / (currentTime - startTime) * 1000 / 1024)
+ + "k/s";
+
+ String leftTime = format((double) (totalLength - readed - downloaded)
+ / readed * (currentTime - startTime) / 1000)
+ + "s";
+
+ String p = format((double) (readed + downloaded)
+ / totalLength * 100)
+ + "%";
+
+ logger.info(p + " all/current [" + speed + "/"
+ + lastSpeed + "] in [" + leftTime + "] "
+ + (readed + downloaded) + "/" + totalLength
+ + "-->" + dest.getName());
+
+ lastTime = currentTime;
+ lastReadPosition = readed;
+ lastPercent = percent;
+ }
+ }
+ if ((readed + downloaded) == totalLength) {
+ logger.info("finished download!--->" + dest);
+ }
+ bos.close();
+ break;
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ }
+}
@@ -0,0 +1,72 @@
+package csstricks;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class DownloadManager {
+
+ public static void main(String[] args) throws MalformedURLException,
+ IOException, InterruptedException {
+ BufferedReader br = new BufferedReader(new FileReader("/tmp/seed.txt"));
+ String line = null;
+ final Download d = new Download();
+ final String template = "http://vnfiles.ign.com/ects/css-tricks/VideoCast-%s.m4v";
+ ExecutorService execs = Executors.newFixedThreadPool(10);
+ while ((line = br.readLine()) != null) {
+ String index = line.split("-")[0];
+ final URL url = new URL(String.format(template, index));
+ final File dest = new File("/home/feng/Downloads/css-tricks/"
+ + line + ".m4v");
+ execs.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ d.download(url, dest);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+ execs.shutdown();
+ execs.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
+ }
+
+ public void start() throws MalformedURLException, IOException {
+ final String prefix = "/video-screencasts/";
+
+ Download d = new Download();
+ Set<String> urLs = d.getURLs(new URL(
+ "http://css-tricks.com/video-screencasts/"));
+ urLs.addAll(d.getURLs(new URL(
+ "http://css-tricks.com/video-screencasts-5/")));
+ urLs.addAll(d.getURLs(new URL(
+ "http://css-tricks.com/video-screencasts-4/")));
+ urLs.addAll(d.getURLs(new URL(
+ "http://css-tricks.com/video-screencasts-3/")));
+ urLs.addAll(d.getURLs(new URL(
+ "http://css-tricks.com/video-screencasts-2/")));
+
+ for (Iterator iterator = urLs.iterator(); iterator.hasNext();) {
+ String u = (String) iterator.next();
+ if (!u.startsWith(prefix) || u.equals(prefix)) {
+ iterator.remove();
+ }
+ }
+
+ for (String u : urLs) {
+ System.out.println(u);
+ }
+
+ System.out.println(urLs.size());
+ }
+}
@@ -0,0 +1,22 @@
+package csstricks;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+public class DownloadUbuntu {
+
+ public static void main(String[] args) throws IOException {
+
+ Download download = new Download();
+
+ File file = new File("/home/feng/Downloads/ubuntu-11-04-alpha3.iso");
+
+ URL url = new URL(
+ "http://cdimage.ubuntu.com/releases/natty/alpha-3/natty-desktop-amd64.iso");
+
+ download.download(url, file);
+
+ }
+
+}

0 comments on commit e4962c7

Please sign in to comment.