init

.gitignore

@@ -0,0 +1,11 @@
+**/target/
+**/.idea/
+java/.settings
+java/.gradle
+**.iml
+**.DS_Store
+**.log
+**/node_modules/
+**/**.pyc
+/awesome/
+/cnn_captcha/

README.md

@@ -0,0 +1,19 @@
+# 代码集合
+
+# javascript
+
+## puppeteer截图
+
+javascript/puppeteer-screenshot.js
+
+使用puppeteer截图，通过滑动到底部来解决图片的懒加载问题。
+
+参考：
+- https://github.com/GoogleChrome/puppeteer/issues/844
+- [如何使用nodejs抓取具有懒加载机制的页面链接](https://github.com/chenxiaochun/blog/issues/35)
+
+# shell
+
+
+
+

douban.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+def main
+
+
+  rand_url = "https://www.douban.com/group/topic/129019947/"
+
+  sleep_seconds = rand(1000)
+  puts "Sleep #{sleep_seconds} for a while."
+  sleep(sleep_seconds)
+
+  case RUBY_PLATFORM
+  when /darwin/
+    `open -g #{rand_url}`
+  when /linux/
+    `xdg-open #{rand_url}`
+  end
+end
+
+if __FILE__ == $0
+  main
+end

java/netty-example/pom.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>cn.morethink</groupId>
+        <artifactId>code</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>netty-example</artifactId>
+    <packaging>jar</packaging>
+    <dependencies>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>4.1.24.Final</version>
+        </dependency>
+        <!--slf4j-->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok-maven-plugin</artifactId>
+            <version>1.18.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.activation</groupId>
+            <artifactId>activation</artifactId>
+            <version>1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.5</version>
+        </dependency>
+    </dependencies>
+
+    <!--<build>-->
+    <!--<plugins>-->
+    <!--<plugin>-->
+    <!--<groupId>org.apache.maven.plugins</groupId>-->
+    <!--<artifactId>maven-compiler-plugin</artifactId>-->
+    <!--<configuration>-->
+    <!--<annotationProcessorPaths>-->
+    <!--<path>-->
+    <!--<groupId>org.mapstruct</groupId>-->
+    <!--<artifactId>mapstruct-processor</artifactId>-->
+    <!--</path>-->
+    <!--<path>-->
+    <!--<groupId>org.projectlombok</groupId>-->
+    <!--<artifactId>lombok</artifactId>-->
+    <!--&lt;!&ndash;选择自己的版本即可&ndash;&gt;-->
+    <!--<version>1.16.20</version>-->
+    <!--</path>-->
+    <!--</annotationProcessorPaths>-->
+    <!--</configuration>-->
+    <!--</plugin>-->
+    <!--</plugins>-->
+    <!--</build>-->
+</project>

java/netty-example/src/main/java/cn/morethink/nettyexample/NettyServer.java

@@ -0,0 +1,41 @@
+package cn.morethink.nettyexample;
+
+import cn.morethink.nettyexample.handler.NettyServerInitializer;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.InetAddress;
+
+/**
+ * @author 李文浩
+ * @date 2018/9/5
+ */
+@Slf4j
+public final class NettyServer {
+
+    public static final int PORT = 9999;
+
+    public static void main(String[] args) throws Exception {
+        EventLoopGroup boss = new NioEventLoopGroup(1);
+        ServerBootstrap b = new ServerBootstrap();
+        b.option(ChannelOption.SO_BACKLOG, 1024);
+        b.group(boss)
+                .channel(NioServerSocketChannel.class)
+                .handler(new LoggingHandler(LogLevel.INFO))
+                //传递路由类
+                .childHandler(new NettyServerInitializer());
+
+        Channel ch = b.bind(PORT).sync().channel();
+        log.info("http://{}:{}/ start", InetAddress.getLocalHost().getHostAddress(), PORT);
+        ch.closeFuture().sync();
+
+    }
+
+}

java/netty-example/src/main/java/cn/morethink/nettyexample/handler/HttUploadHandler.java

@@ -0,0 +1,100 @@
+package cn.morethink.nettyexample.handler;
+
+import cn.morethink.nettyexample.util.GeneralResponse;
+import cn.morethink.nettyexample.util.ResponseUtil;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.*;
+import io.netty.handler.codec.http.multipart.*;
+import io.netty.util.ReferenceCountUtil;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+
+import static io.netty.handler.codec.http.multipart.InterfaceHttpData.HttpDataType;
+
+/**
+ * 接收HTTP文件上传的处理器
+ *
+ * @author 李文浩
+ * @date 2018/10/11
+ */
+@Slf4j
+public class HttUploadHandler extends SimpleChannelInboundHandler<HttpObject> {
+
+    public HttUploadHandler() {
+        super(false);
+    }
+
+    private static final HttpDataFactory factory = new DefaultHttpDataFactory(true);
+    private static final String FILE_UPLOAD = "/data/";
+    private static final String URI = "/upload";
+    private HttpPostRequestDecoder httpDecoder;
+    HttpRequest request;
+
+    @Override
+    protected void channelRead0(final ChannelHandlerContext ctx, final HttpObject httpObject)
+            throws Exception {
+        if (httpObject instanceof HttpRequest) {
+            request = (HttpRequest) httpObject;
+            if (request.uri().startsWith(URI) && request.method().equals(HttpMethod.POST)) {
+                httpDecoder = new HttpPostRequestDecoder(factory, request);
+                httpDecoder.setDiscardThreshold(0);
+            } else {
+                //传递给下一个Handler
+                ctx.fireChannelRead(httpObject);
+            }
+        }
+        if (httpObject instanceof HttpContent) {
+            if (httpDecoder != null) {
+                final HttpContent chunk = (HttpContent) httpObject;
+                httpDecoder.offer(chunk);
+                if (chunk instanceof LastHttpContent) {
+                    writeChunk(ctx);
+                    //关闭httpDecoder
+                    httpDecoder.destroy();
+                    httpDecoder = null;
+                }
+                ReferenceCountUtil.release(httpObject);
+            } else {
+                ctx.fireChannelRead(httpObject);
+            }
+        }
+
+    }
+
+    private void writeChunk(ChannelHandlerContext ctx) throws IOException {
+        while (httpDecoder.hasNext()) {
+            InterfaceHttpData data = httpDecoder.next();
+            if (data != null && HttpDataType.FileUpload.equals(data.getHttpDataType())) {
+                final FileUpload fileUpload = (FileUpload) data;
+                final File file = new File(FILE_UPLOAD + fileUpload.getFilename());
+                log.info("upload file: {}", file);
+                try (FileChannel inputChannel = new FileInputStream(fileUpload.getFile()).getChannel();
+                     FileChannel outputChannel = new FileOutputStream(file).getChannel()) {
+                    outputChannel.transferFrom(inputChannel, 0, inputChannel.size());
+                    ResponseUtil.response(ctx, request, new GeneralResponse(HttpResponseStatus.OK, "SUCCESS", null));
+                }
+            }
+        }
+    }
+
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+        log.warn("{}", cause);
+        ctx.channel().close();
+    }
+
+    @Override
+    public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+        if (httpDecoder != null) {
+            httpDecoder.cleanFiles();
+        }
+    }
+
+}

java/netty-example/src/main/java/cn/morethink/nettyexample/handler/HttpDownloadHandler.java

@@ -0,0 +1,84 @@
+package cn.morethink.nettyexample.handler;
+
+import cn.morethink.nettyexample.util.GeneralResponse;
+import cn.morethink.nettyexample.util.ResponseUtil;
+import io.netty.channel.*;
+import io.netty.handler.codec.http.*;
+import io.netty.handler.stream.ChunkedFile;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+
+/**
+ * @author 李文浩
+ * @date 2018/9/18
+ */
+@Slf4j
+public class HttpDownloadHandler extends SimpleChannelInboundHandler<FullHttpRequest> {
+
+    public HttpDownloadHandler() {
+        super(false);
+    }
+
+    private String filePath = "/data/body.csv";
+
+    @Override
+    protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) {
+        String uri = request.uri();
+        if (uri.startsWith("/download") && request.method().equals(HttpMethod.GET)) {
+            GeneralResponse generalResponse = null;
+            File file = new File(filePath);
+            try {
+                final RandomAccessFile raf = new RandomAccessFile(file, "r");
+                long fileLength = raf.length();
+                HttpResponse response = new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.OK);
+                response.headers().set(HttpHeaderNames.CONTENT_LENGTH, fileLength);
+                response.headers().set(HttpHeaderNames.CONTENT_TYPE, "application/octet-stream");
+                response.headers().add(HttpHeaderNames.CONTENT_DISPOSITION, String.format("attachment; filename=\"%s\"", file.getName()));
+                ctx.write(response);
+                ChannelFuture sendFileFuture = ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise());
+                sendFileFuture.addListener(new ChannelProgressiveFutureListener() {
+                    @Override
+                    public void operationComplete(ChannelProgressiveFuture future)
+                            throws Exception {
+                        log.info("file {} transfer complete.", file.getName());
+                        raf.close();
+                    }
+
+                    @Override
+                    public void operationProgressed(ChannelProgressiveFuture future,
+                                                    long progress, long total) throws Exception {
+                        if (total < 0) {
+                            log.warn("file {} transfer progress: {}", file.getName(), progress);
+                        } else {
+                            log.debug("file {} transfer progress: {}/{}", file.getName(), progress, total);
+                        }
+                    }
+                });
+                ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
+            } catch (FileNotFoundException e) {
+                log.warn("file {} not found", file.getPath());
+                generalResponse = new GeneralResponse(HttpResponseStatus.NOT_FOUND, String.format("file %s not found", file.getPath()), null);
+                ResponseUtil.response(ctx, request, generalResponse);
+            } catch (IOException e) {
+                log.warn("file {} has a IOException: {}", file.getName(), e.getMessage());
+                generalResponse = new GeneralResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, String.format("读取 file %s 发生异常", filePath), null);
+                ResponseUtil.response(ctx, request, generalResponse);
+            }
+        } else {
+            ctx.fireChannelRead(request);
+        }
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) {
+        log.warn("{}", e);
+        ctx.close();
+
+    }
+}

java/netty-example/src/main/java/cn/morethink/nettyexample/handler/NettyServerHandler.java

@@ -0,0 +1,34 @@
+package cn.morethink.nettyexample.handler;
+
+
+import cn.morethink.nettyexample.util.GeneralResponse;
+import cn.morethink.nettyexample.util.ResponseUtil;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @author 李文浩
+ * @date 2018/9/5
+ */
+@Slf4j
+public class NettyServerHandler extends SimpleChannelInboundHandler<FullHttpRequest> {
+
+
+    @Override
+    public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) {
+        GeneralResponse generalResponse;
+        //错误处理
+        generalResponse = new GeneralResponse(HttpResponseStatus.BAD_REQUEST, "请检查你的请求方法及url", null);
+        ResponseUtil.response(ctx, request, generalResponse);
+
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) {
+        log.warn("{}", e);
+        ctx.close();
+    }
+}