Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

增加了计算MD5值去重。综合考虑的几种容器,对md5的操作使用了set,在插入和查找MD5值都具有较高的效率,因为它是基于红黑树

  • Loading branch information...
commit 7494959c423e58cff253ada8920b8a4e2340c23f 1 parent 62c10ef
Yemsheng authored
Showing with 420 additions and 6 deletions.
  1. +368 −0 spider/md5.h
  2. +52 −6 spider/spider.cpp
368 spider/md5.h
View
@@ -0,0 +1,368 @@
+
+/*
+* MD5算法来自 http://blog.csdn.net/sdsky1987/article/details/6660771
+*
+*
+*/
+
+#ifndef _MD5_H_
+#define _MD5_H_
+
+char* MD5String( char* string );
+bool MD5Check( char *md5string, char* string );
+
+#endif //_MD5_H_
+
+
+#include<iostream>
+#include<stdio.h>
+#include<stdlib.h>
+#include<string.h>
+#include<math.h>
+#include<ctype.h>
+
+typedef unsigned char *POINTER;
+typedef unsigned short int UINT2;
+typedef unsigned long int UINT4;
+
+#define PROTO_LIST(list) list
+
+#define S11 7
+#define S12 12
+#define S13 17
+#define S14 22
+#define S21 5
+#define S22 9
+#define S23 14
+#define S24 20
+#define S31 4
+#define S32 11
+#define S33 16
+#define S34 23
+#define S41 6
+#define S42 10
+#define S43 15
+#define S44 21
+
+#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
+#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
+#define H(x, y, z) ((x) ^ (y) ^ (z))
+#define I(x, y, z) ((y) ^ ((x) | (~z)))
+
+typedef struct _MD5_CTX
+{
+ UINT4 state[4];
+ UINT4 count[2];
+ unsigned char buffer[64];
+} MD5_CTX;
+
+static unsigned char PADDING[64] = {
+ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
+#define FF(a, b, c, d, x, s, ac) { \
+ (a) += F ((b), (c), (d)) + (x) + (UINT4)(ac);\
+ (a) = ROTATE_LEFT ((a), (s)); \
+ (a) += (b); \
+ }
+#define GG(a, b, c, d, x, s, ac) { \
+ (a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \
+ (a) = ROTATE_LEFT ((a), (s)); \
+ (a) += (b); \
+ }
+#define HH(a, b, c, d, x, s, ac) { \
+ (a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \
+ (a) = ROTATE_LEFT ((a), (s)); \
+ (a) += (b); \
+ }
+#define II(a, b, c, d, x, s, ac) { \
+ (a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \
+ (a) = ROTATE_LEFT ((a), (s)); \
+ (a) += (b); \
+ }
+
+#define TEST_BLOCK_LEN 1000
+#define TEST_BLOCK_COUNT 1000
+
+static void MD5Transform PROTO_LIST ((UINT4 [4], unsigned char [64]));
+static void Encode PROTO_LIST ((unsigned char *, UINT4 *, unsigned int));
+static void Decode PROTO_LIST ((UINT4 *, unsigned char *, unsigned int));
+static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int));
+static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int));
+static void MD5Init PROTO_LIST ((MD5_CTX *));
+static void MD5Update PROTO_LIST ((MD5_CTX *, unsigned char *, unsigned int));
+static void MD5Final PROTO_LIST ((unsigned char [16], MD5_CTX *));
+static void MDTimeTrial PROTO_LIST ((void));
+static void StringAddOne PROTO_LIST ((char *));
+static void Encode PROTO_LIST ((unsigned char *, UINT4 *, unsigned int));
+static void Decode PROTO_LIST ((UINT4 *, unsigned char *, unsigned int));
+
+static void MD5Init( MD5_CTX *context )
+{
+ context->count[0] = context->count[1] = 0;
+ context->state[0] = 0x67452301;
+ context->state[1] = 0xefcdab89;
+ context->state[2] = 0x98badcfe;
+ context->state[3] = 0x10325476;
+}
+
+static void MD5Update(
+ MD5_CTX *context, /* context */
+ unsigned char *input, /* input block */
+ unsigned int inputLen /* length of input block */
+ )
+{
+ unsigned int i, index, partLen;
+
+ index = (unsigned int)((context->count[0] >> 3) & 0x3F);
+
+ if ((context->count[0] += ((UINT4)inputLen << 3))
+ < ((UINT4)inputLen << 3))
+ context->count[1]++;
+ context->count[1] += ((UINT4)inputLen >> 29);
+
+ partLen = 64 - index;
+
+ if (inputLen >= partLen) {
+ MD5_memcpy
+ ((POINTER)&context->buffer[index], (POINTER)input, partLen);
+ MD5Transform (context->state, context->buffer);
+
+ for (i = partLen; i + 63 < inputLen; i += 64)
+ MD5Transform (context->state, &input[i]);
+
+ index = 0;
+ }
+ else
+ i = 0;
+
+ MD5_memcpy
+ ((POINTER)&context->buffer[index], (POINTER)&input[i],
+ inputLen-i);
+}
+
+static void MD5Final(
+ unsigned char digest[16], /* message digest */
+ MD5_CTX *context /* context */
+ )
+{
+ unsigned char bits[8];
+ unsigned int index, padLen;
+
+ Encode (bits, context->count, 8);
+
+ index = (unsigned int)((context->count[0] >> 3) & 0x3f);
+ padLen = (index < 56) ? (56 - index) : (120 - index);
+ MD5Update (context, PADDING, padLen);
+
+ MD5Update (context, bits, 8);
+
+ Encode (digest, context->state, 16);
+
+ MD5_memset ((POINTER)context, 0, sizeof (*context));
+}
+
+static void MD5Transform(
+ UINT4 state[4],
+ unsigned char block[64]
+ )
+{
+ UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16];
+
+ Decode (x, block, 64);
+
+ /* Round 1 */
+ FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
+ FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
+ FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
+ FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
+ FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
+ FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
+ FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
+ FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
+ FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
+ FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
+ FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
+ FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
+ FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
+ FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
+ FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
+ FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
+
+ /* Round 2 */
+ GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
+ GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
+ GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
+ GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
+ GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
+ GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */
+ GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
+ GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
+ GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
+ GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
+ GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
+ GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
+ GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
+ GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
+ GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
+ GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
+
+ /* Round 3 */
+ HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
+ HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
+ HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
+ HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
+ HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
+ HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
+ HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
+ HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
+ HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
+ HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
+ HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
+ HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */
+ HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
+ HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
+ HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
+ HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
+
+ /* Round 4 */
+ II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
+ II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
+ II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
+ II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
+ II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
+ II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
+ II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
+ II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
+ II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
+ II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
+ II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
+ II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
+ II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
+ II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
+ II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
+ II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
+
+ state[0] += a;
+ state[1] += b;
+ state[2] += c;
+ state[3] += d;
+
+ MD5_memset ((POINTER)x, 0, sizeof (x));
+}
+
+static void Encode(
+ unsigned char *output,
+ UINT4 *input,
+ unsigned int len
+ )
+{
+ unsigned int i, j;
+
+ for (i = 0, j = 0; j < len; i++, j += 4) {
+ output[j] = (unsigned char)(input[i] & 0xff);
+ output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
+ output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
+ output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
+ }
+}
+
+static void Decode(
+ UINT4 *output,
+ unsigned char *input,
+ unsigned int len
+ )
+{
+ unsigned int i, j;
+
+ for (i = 0, j = 0; j < len; i++, j += 4)
+ output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) |
+ (((UINT4)input[j+2]) << 16) | (((UINT4)input[j+3]) << 24);
+}
+
+static void MD5_memcpy(
+ POINTER output,
+ POINTER input,
+ unsigned int len
+ )
+{
+ unsigned int i;
+
+ for (i = 0; i < len; i++)
+ output[i] = input[i];
+}
+
+static void MD5_memset(
+ POINTER output,
+ int value,
+ unsigned int len
+ )
+{
+ unsigned int i;
+
+ for (i = 0; i < len; i++)
+ ((char *)output)[i] = (char)value;
+}
+
+char* MD5String( char *string )
+{
+ MD5_CTX context;
+ unsigned char digest[16];
+ char output1[33];
+ static char output[33]={""};
+ unsigned int len = strlen (string);
+ int i;
+
+ MD5Init( &context);
+ MD5Update( &context, (unsigned char*)string, len );
+ MD5Final( digest, &context );
+
+ for (i = 0; i < 16; i++)
+ {
+ sprintf(&(output1[2*i]),"%02x",(unsigned char)digest[i]);
+ sprintf(&(output1[2*i+1]),"%02x",(unsigned char)(digest[i]<<4));
+ }
+ for(i=0;i<32;i++)
+ {
+ output[i]=output1[i];
+ }
+ return output;
+}
+
+static void StringAddOne( char * orstring )
+{
+ unsigned int len;
+ int i,n;
+
+ len = strlen(orstring);
+ n = len - 1;
+ for(i = n; i >= 0; i--)
+ {
+ if(orstring[i]=='9')
+ {
+ orstring[i] = 'A';
+ break;
+ }
+ else if(orstring[i]=='Z')
+ {
+ orstring[i]='a';
+ break;
+ }
+ else if(orstring[i]=='z')
+ {
+ orstring[i]='0';
+ continue;
+ }
+ else
+ orstring[i] += 1;
+ break;
+ }
+}
+
+bool MD5Check( char *md5string, char* string )
+{
+ return strcmp( md5string, MD5String( string ) ) == 0;
+}
58 spider/spider.cpp
View
@@ -19,8 +19,11 @@
#include <queue>
#include <string>
+#include <set>
using namespace std;
+#include "md5.h"
+
const int URL_SIZE = 512;
const int DOMAIN_SIZE = 128;
const int FILE_NAME_SIZE = 512;
@@ -34,6 +37,9 @@ char *getDomainFromUrl(const char *url, char *domainBuf, int domainBufSize);
bool spiderOnePage(const char *domain, const char *url, const char* saveFileName);
bool analyzeOnePage(const char *fileName, queue<string> *q_url, bool spiderFlag);
char *getSaveFileName(char *fileNameBuf, const int NameBufSize);
+bool isUrlMd5Exist(const char *urlMd5);
+
+set<string> g_urlMd5Set;
int main(int agrc, char **argv)
@@ -42,6 +48,8 @@ int main(int agrc, char **argv)
queue<string> q_url;
string firstPage = "http://www.dangdang.com";
q_url.push(firstPage);
+ char *firstPageMd5str = MD5String((char*)firstPage.c_str());
+ g_urlMd5Set.insert(string(firstPageMd5str));
string url;
char domainBuffer[DOMAIN_SIZE];
@@ -243,12 +251,20 @@ char *getDomainFromUrl(const char *url, char *domainBuf, int domainBufSize)
fprintf(stdout, "getDomainFromUrl function\n");
if(url==NULL||domainBuf==NULL)
+ {
+ fprintf(stderr, "getDomainFromUrl url==NULL||domainBuf==NULL return NULL\n");
return NULL;
-
+ }
char *httpProtocol = "http://";
const int urlLen = strlen(url);
const int protocolLen = strlen(httpProtocol);
+ if(urlLen<=protocolLen)
+ {
+ fprintf(stderr, "getDomainFromUrl urlLen<=protocolLen return NULL\n");
+ fprintf(stderr, "getDomainFromUrl url = %s\n", url);
+ return NULL;
+ }
char *domainEnd = "/";
char *isFind = strstr((char*)(url+protocolLen), domainEnd);
int domainLen = 0;
@@ -262,11 +278,15 @@ char *getDomainFromUrl(const char *url, char *domainBuf, int domainBufSize)
}
if(domainLen>=domainBufSize)
- return NULL;
+ {
+ fprintf(stderr, "getDomainFromUrl domainLen>=domainBufSize return NULL\n");
+ return NULL;
+ }
strncpy(domainBuf, url+protocolLen, domainLen);
domainBuf[domainLen] = '\0';
-
+
+ fprintf(stdout, "getDomainFromUrl domainBuf = %s\n", domainBuf);
return domainBuf;
}
@@ -331,12 +351,26 @@ void analyzeAndAddQueue(FILE** fread_h, queue<string> *q_url)
strncpy(urlBuf, httpFindIndex, len);
urlBuf[len] = '\0';
- if(!hasDomain(urlBuf, "dangdang"))
+ char domainBuf[DOMAIN_SIZE];
+ memset(domainBuf, 0, sizeof(domainBuf));
+ char *getDomainResult = getDomainFromUrl(urlBuf, domainBuf, sizeof(domainBuf));
+ if(getDomainResult==NULL)
+ continue;
+
+ if(!hasDomain(domainBuf, "dangdang"))
continue;
- printf("find an url: %s\n", urlBuf);
+ printf("\nfind an url: %s\n", urlBuf);
+ char *urlMd5str = MD5String(urlBuf);
+ printf("find an md5: %s\n", urlMd5str);
+
//add to the queue
- q_url->push(urlBuf);
+ if(!isUrlMd5Exist(urlMd5str))
+ {
+ g_urlMd5Set.insert(string(urlMd5str));
+ q_url->push(urlBuf);
+ printf("push an url in queue: %s\n\n", urlBuf);
+ }
}
}
@@ -347,6 +381,18 @@ void analyzeAndAddQueue(FILE** fread_h, queue<string> *q_url)
/**
+*判断一个url是否已经存在
+*/
+bool isUrlMd5Exist(const char *urlMd5)
+{
+ string urlMd5Str(urlMd5);
+ const bool isUrlIn = g_urlMd5Set.find(urlMd5Str)!=g_urlMd5Set.end();
+
+ return isUrlIn;
+}
+
+
+/**
*检查url中是否有客户(举例:dangdang)
*/
bool hasDomain(char *url, char *client)
Please sign in to comment.
Something went wrong with that request. Please try again.