multithread download, download resume, used to download video from cs…

…s-tricks.com

src/main/resources/seed.txt

@@ -0,0 +1,79 @@
+15-introduction-to-firebug
+39-how-to-use-sifr-3
+35-intro-to-jquery-2
+74-editable-css3-image-gallery
+59-embedding-audio
+28-using-wufoo-for-web-forms
+60-ajax-refreshing-rss-content
+43-how-to-use-css-sprites
+70-random-pet-peeves
+17-sliding-doors-button
+38-basics-tips-on-designing-for-the-iphone
+29-google-search-for-your-site
+91-the-wordpress-loop
+22-cutting-clipping-paths
+25-designing-for-wordpress-part-one
+67-jquery-part-3-image-title-plugin
+86-mamp
+33-ecommerce-product-page-makeover
+23-conditional-stylesheets
+93-css3-slideup-boxes
+13-converting-a-photoshop-mockup-part-two-episode-two
+31-introduction-to-google-analytics
+63-on-screencasting
+18-introduction-to-the-band-website-template
+76-jquery-tour
+85-best-practices-dynamic-content
+36-current-nav-highlighting-using-php-to-set-the-body-id
+65-advanced-uses-for-custom-fields-in-wordpress
+71-building-a-website-photoshop-mockup
+8-css-formatting
+64-building-a-photo-gallery
+4-forcing-scrollbars-eliminating-horizonatal-jumps
+66-table-styling-2-fixed-header-and-highlighting
+5-columns-of-equal-height-super-simple-two-column-layout
+40-how-z-index-works
+34-integrating-and-customizing-google-maps
+42-all-about-floats-screencast
+83-thoughts-on-seo
+61-basic-table-styling
+79-complete-jquery-animations
+81-ajaxing-a-wordpress-theme
+90-simple-textmate-tips
+14-converting-a-photoshop-mockup-part-two-episode-three
+41-wordpress-as-a-cms
+19-designing-a-unique-page-for-twitter-updates
+84-site-walkthrough
+77-styling-an-individual-article
+3-converting-a-photoshop-mockup-part-3-of-3
+73-wordpress-theme
+88-intro-to-compass-sass
+94-intro-to-pseudo-elements
+58-html-css-the-very-basics
+78-on-web-advertising
+87-moving-up-with-mamp
+89-organizing-photoshop
+37-status-fluid-and-menu-bar-apps
+92-drawing-table
+16-creating-the-photoshop-mockup
+20-introduction-to-jquery
+11-working-modularly-with-php
+68-think-geek-background-fade-technique
+9-starry-night-3d-background-with-the-parallax-effect
+69-first-ten-minutes-with-typekit
+30-creating-and-sending-html-email
+24-rounded-corners
+32-using-the-unit-png-fix
+72-building-a-website-2-of-3-htmlcss-conversion
+2-converting-a-photoshop-mockup-part-2-of-3
+75-how-not-to-design-a-checkout
+80-regarding-wheel-invention
+7-three-state-menu
+62-advanced-form-styling-functionality
+82-css-image-switcher
+12-converting-a-photoshop-mockup-part-two-episode-one
+10-fixed-width-fluid-width-elastic-width
+27-designing-for-wordpress-part-three
+21-walkthrough-of-contact-form
+26-designing-for-wordpress-part-two
+6-tools-of-the-trade

src/test/java/csstricks/CountDownLatchTest.java

@@ -0,0 +1,43 @@
+package csstricks;
+
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class CountDownLatchTest {
+
+	private static class MyRunnable implements Runnable {
+
+		private CountDownLatch latch;
+
+		public MyRunnable(CountDownLatch latch) {
+			this.latch = latch;
+		}
+
+		@Override
+		public void run() {
+			try {
+				TimeUnit.MILLISECONDS.sleep(new Random().nextInt(1000));
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+			latch.countDown();
+//			latch.
+			System.out.println("over");
+		}
+
+	}
+
+	public static void main(String[] args) throws InterruptedException {
+
+		int SIZE = 3;
+		CountDownLatch latch = new CountDownLatch(SIZE);
+		for (int i = 0; i < SIZE; i++) {
+			new Thread(new MyRunnable(latch)).start();
+		}
+
+		latch.await();
+		System.out.println("all finished");
+
+	}
+}

src/test/java/csstricks/Download.java

@@ -0,0 +1,159 @@
+package csstricks;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.DecimalFormat;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+
+public class Download {
+	private static Logger logger = Logger.getLogger(Download.class);
+
+	public static void main(String[] args) throws IOException {
+		Download manager = new Download();
+		// String
+		manager.download(new URL(
+				"http://files/java/eclipse-jee-helios-SR1-linux-gtk.tar.gz"),
+				new File("/tmp/what.gif"));
+	}
+
+	public Set<String> getURLs(URL url) throws IOException {
+		InputStream stream = url.openConnection().getInputStream();
+		BufferedReader br = new BufferedReader(new InputStreamReader(stream));
+		String line = null;
+		StringBuilder sb = new StringBuilder();
+		while ((line = br.readLine()) != null) {
+			sb.append(line).append("\n");
+		}
+		// System.out.println(sb.toString());
+		Set<String> urls = new HashSet<String>();
+
+		Pattern p = Pattern.compile("\\shref=\"(.+?)\"");
+		Matcher matcher = p.matcher(sb);
+		while (matcher.find()) {
+			String u = matcher.group(1);
+			urls.add(u);
+		}
+		return urls;
+	}
+
+	public long getLength(URL url) throws IOException {
+		HttpURLConnection con = (HttpURLConnection) url.openConnection();
+		con.setRequestMethod("HEAD");
+		return Long.parseLong(con.getHeaderField("Content-Length"));
+	}
+
+	private static final DecimalFormat df = new DecimalFormat("##.##");
+
+	private String percent(long total, long finished) {
+		double p = (double) finished / total * 100;
+		return df.format(p);
+	}
+
+	private static String format(double d) {
+		return df.format(d);
+	}
+
+	public void download(URL url, File dest) throws IOException {
+		logger.info(url + "-->" + dest);
+		for (int i = 0; i < 30; ++i) {
+			try {
+				HttpURLConnection con = (HttpURLConnection) url
+						.openConnection();
+				long readed = 0; // bytes this time download
+				long totalLength = getLength(url); // total length of the
+													// resource
+				long downloaded = 0; // already downloaded before this try
+				double lastPercent = 0.0;
+				if (dest.exists()) {
+					downloaded = dest.length();
+					if (totalLength == downloaded) {
+						logger.info("already finished, exits");
+						return;
+					} else if (totalLength < downloaded) {
+						logger.error(downloaded + " > " + totalLength
+								+ "; error!");
+						return;
+					}
+					con.addRequestProperty("range", "bytes=" + downloaded + "-"
+							+ totalLength);
+					logger.info("resume download; " + downloaded + "/"
+							+ totalLength + "; "
+							+ percent(totalLength, downloaded));
+					lastPercent = (double) downloaded / totalLength;
+				}
+				con.connect();
+
+				Map<String, List<String>> headers = con.getHeaderFields();
+				for (Entry<String, List<String>> e : headers.entrySet()) {
+					logger.info(e.getKey() + ": " + e.getValue());
+				}
+
+				InputStream stream = con.getInputStream();
+				BufferedOutputStream bos = new BufferedOutputStream(
+						new FileOutputStream(dest, true));
+				long startTime = System.currentTimeMillis();
+				long lastReadPosition = readed;
+				long lastTime = startTime;
+
+				byte[] buffer = new byte[8912 * 2];
+				int read = -1;
+
+				while ((read = stream.read(buffer)) != -1) {
+					readed += read;
+					bos.write(buffer, 0, read);
+					double percent = (double) (readed + downloaded)
+							/ totalLength;
+					if (percent - lastPercent > 0.01) {
+						long currentTime = System.currentTimeMillis();
+						String lastSpeed = format((double) (readed - lastReadPosition)
+								/ (currentTime - lastTime) * 1000 / 1024)
+								+ "k/s";
+
+						String speed = format((double) readed
+								/ (currentTime - startTime) * 1000 / 1024)
+								+ "k/s";
+
+						String leftTime = format((double) (totalLength - readed - downloaded)
+								/ readed * (currentTime - startTime) / 1000)
+								+ "s";
+
+						String p = format((double) (readed + downloaded)
+								/ totalLength * 100)
+								+ "%";
+
+						logger.info(p + " all/current [" + speed + "/"
+								+ lastSpeed + "] in [" + leftTime + "] "
+								+ (readed + downloaded) + "/" + totalLength
+								+ "-->" + dest.getName());
+
+						lastTime = currentTime;
+						lastReadPosition = readed;
+						lastPercent = percent;
+					}
+				}
+				if ((readed + downloaded) == totalLength) {
+					logger.info("finished download!--->" + dest);
+				}
+				bos.close();
+				break;
+			} catch (Exception e) {
+				logger.error(e.getMessage(), e);
+			}
+		}
+	}
+}

src/test/java/csstricks/DownloadManager.java

@@ -0,0 +1,72 @@
+package csstricks;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class DownloadManager {
+
+	public static void main(String[] args) throws MalformedURLException,
+			IOException, InterruptedException {
+		BufferedReader br = new BufferedReader(new FileReader("/tmp/seed.txt"));
+		String line = null;
+		final Download d = new Download();
+		final String template = "http://vnfiles.ign.com/ects/css-tricks/VideoCast-%s.m4v";
+		ExecutorService execs = Executors.newFixedThreadPool(10);
+		while ((line = br.readLine()) != null) {
+			String index = line.split("-")[0];
+			final URL url = new URL(String.format(template, index));
+			final File dest = new File("/home/feng/Downloads/css-tricks/"
+					+ line + ".m4v");
+			execs.submit(new Runnable() {
+				@Override
+				public void run() {
+					try {
+						d.download(url, dest);
+					} catch (IOException e) {
+						e.printStackTrace();
+					}
+				}
+			});
+		}
+		execs.shutdown();
+		execs.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
+	}
+
+	public void start() throws MalformedURLException, IOException {
+		final String prefix = "/video-screencasts/";
+
+		Download d = new Download();
+		Set<String> urLs = d.getURLs(new URL(
+				"http://css-tricks.com/video-screencasts/"));
+		urLs.addAll(d.getURLs(new URL(
+				"http://css-tricks.com/video-screencasts-5/")));
+		urLs.addAll(d.getURLs(new URL(
+				"http://css-tricks.com/video-screencasts-4/")));
+		urLs.addAll(d.getURLs(new URL(
+				"http://css-tricks.com/video-screencasts-3/")));
+		urLs.addAll(d.getURLs(new URL(
+				"http://css-tricks.com/video-screencasts-2/")));
+
+		for (Iterator iterator = urLs.iterator(); iterator.hasNext();) {
+			String u = (String) iterator.next();
+			if (!u.startsWith(prefix) || u.equals(prefix)) {
+				iterator.remove();
+			}
+		}
+
+		for (String u : urLs) {
+			System.out.println(u);
+		}
+
+		System.out.println(urLs.size());
+	}
+}

src/test/java/csstricks/DownloadUbuntu.java

@@ -0,0 +1,22 @@
+package csstricks;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+public class DownloadUbuntu {
+
+	public static void main(String[] args) throws IOException {
+
+		Download download = new Download();
+
+		File file = new File("/home/feng/Downloads/ubuntu-11-04-alpha3.iso");
+
+		URL url = new URL(
+				"http://cdimage.ubuntu.com/releases/natty/alpha-3/natty-desktop-amd64.iso");
+
+		download.download(url, file);
+
+	}
+
+}