diff --git a/.gitignore b/.gitignore index ebda3df..32ad6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore output of scraper data.sqlite +node_modules *.pyc *.sublime-workspace CartolaFCScraper.sublime-project diff --git a/Data/Partidas.json b/Data/Partidas.json deleted file mode 100644 index 368cd86..0000000 --- a/Data/Partidas.json +++ /dev/null @@ -1,1522 +0,0 @@ -[ - { - "ID": 190053, - "Rodada": 1 - }, - { - "ID": 190054, - "Rodada": 1 - }, - { - "ID": 190055, - "Rodada": 1 - }, - { - "ID": 190056, - "Rodada": 1 - }, - { - "ID": 190057, - "Rodada": 1 - }, - { - "ID": 190058, - "Rodada": 1 - }, - { - "ID": 190059, - "Rodada": 1 - }, - { - "ID": 190060, - "Rodada": 1 - }, - { - "ID": 190061, - "Rodada": 1 - }, - { - "ID": 190062, - "Rodada": 1 - }, - { - "ID": 190063, - "Rodada": 2 - }, - { - "ID": 190064, - "Rodada": 2 - }, - { - "ID": 190065, - "Rodada": 2 - }, - { - "ID": 190066, - "Rodada": 2 - }, - { - "ID": 190067, - "Rodada": 2 - }, - { - "ID": 190068, - "Rodada": 2 - }, - { - "ID": 190069, - "Rodada": 2 - }, - { - "ID": 190070, - "Rodada": 2 - }, - { - "ID": 190071, - "Rodada": 2 - }, - { - "ID": 190072, - "Rodada": 2 - }, - { - "ID": 190073, - "Rodada": 3 - }, - { - "ID": 190074, - "Rodada": 3 - }, - { - "ID": 190075, - "Rodada": 3 - }, - { - "ID": 190076, - "Rodada": 3 - }, - { - "ID": 190077, - "Rodada": 3 - }, - { - "ID": 190078, - "Rodada": 3 - }, - { - "ID": 190079, - "Rodada": 3 - }, - { - "ID": 190080, - "Rodada": 3 - }, - { - "ID": 190081, - "Rodada": 3 - }, - { - "ID": 190082, - "Rodada": 3 - }, - { - "ID": 190083, - "Rodada": 4 - }, - { - "ID": 190084, - "Rodada": 4 - }, - { - "ID": 190085, - "Rodada": 4 - }, - { - "ID": 190086, - "Rodada": 4 - }, - { - "ID": 190087, - "Rodada": 4 - }, - { - "ID": 190088, - "Rodada": 4 - }, - { - "ID": 190089, - "Rodada": 4 - }, - { - "ID": 190090, - "Rodada": 4 - }, - { - "ID": 190091, - "Rodada": 4 - }, - { - "ID": 190092, - "Rodada": 4 - }, - { - "ID": 190093, - "Rodada": 5 - }, - { - "ID": 190094, - "Rodada": 5 - }, - { - "ID": 190095, - "Rodada": 5 - }, - { - "ID": 190096, - "Rodada": 5 - }, - { - "ID": 190097, - "Rodada": 5 - }, - { - "ID": 190098, - "Rodada": 5 - }, - { - "ID": 190099, - "Rodada": 5 - }, - { - "ID": 190100, - "Rodada": 5 - }, - { - "ID": 190101, - "Rodada": 5 - }, - { - "ID": 190102, - "Rodada": 5 - }, - { - "ID": 190103, - "Rodada": 6 - }, - { - "ID": 190104, - "Rodada": 6 - }, - { - "ID": 190105, - "Rodada": 6 - }, - { - "ID": 190106, - "Rodada": 6 - }, - { - "ID": 190107, - "Rodada": 6 - }, - { - "ID": 190108, - "Rodada": 6 - }, - { - "ID": 190109, - "Rodada": 6 - }, - { - "ID": 190110, - "Rodada": 6 - }, - { - "ID": 190111, - "Rodada": 6 - }, - { - "ID": 190112, - "Rodada": 6 - }, - { - "ID": 190113, - "Rodada": 7 - }, - { - "ID": 190114, - "Rodada": 7 - }, - { - "ID": 190115, - "Rodada": 7 - }, - { - "ID": 190116, - "Rodada": 7 - }, - { - "ID": 190117, - "Rodada": 7 - }, - { - "ID": 190118, - "Rodada": 7 - }, - { - "ID": 190119, - "Rodada": 7 - }, - { - "ID": 190120, - "Rodada": 7 - }, - { - "ID": 190121, - "Rodada": 7 - }, - { - "ID": 190122, - "Rodada": 7 - }, - { - "ID": 190123, - "Rodada": 8 - }, - { - "ID": 190124, - "Rodada": 8 - }, - { - "ID": 190125, - "Rodada": 8 - }, - { - "ID": 190126, - "Rodada": 8 - }, - { - "ID": 190127, - "Rodada": 8 - }, - { - "ID": 190128, - "Rodada": 8 - }, - { - "ID": 190129, - "Rodada": 8 - }, - { - "ID": 190130, - "Rodada": 8 - }, - { - "ID": 190131, - "Rodada": 8 - }, - { - "ID": 190132, - "Rodada": 8 - }, - { - "ID": 190133, - "Rodada": 9 - }, - { - "ID": 190134, - "Rodada": 9 - }, - { - "ID": 190135, - "Rodada": 9 - }, - { - "ID": 190136, - "Rodada": 9 - }, - { - "ID": 190137, - "Rodada": 9 - }, - { - "ID": 190138, - "Rodada": 9 - }, - { - "ID": 190140, - "Rodada": 9 - }, - { - "ID": 190141, - "Rodada": 9 - }, - { - "ID": 190142, - "Rodada": 9 - }, - { - "ID": 190143, - "Rodada": 9 - }, - { - "ID": 190144, - "Rodada": 10 - }, - { - "ID": 190145, - "Rodada": 10 - }, - { - "ID": 190146, - "Rodada": 10 - }, - { - "ID": 190147, - "Rodada": 10 - }, - { - "ID": 190148, - "Rodada": 10 - }, - { - "ID": 190149, - "Rodada": 10 - }, - { - "ID": 190150, - "Rodada": 10 - }, - { - "ID": 190151, - "Rodada": 10 - }, - { - "ID": 190152, - "Rodada": 10 - }, - { - "ID": 190153, - "Rodada": 10 - }, - { - "ID": 190154, - "Rodada": 11 - }, - { - "ID": 190155, - "Rodada": 11 - }, - { - "ID": 190156, - "Rodada": 11 - }, - { - "ID": 190157, - "Rodada": 11 - }, - { - "ID": 190158, - "Rodada": 11 - }, - { - "ID": 190159, - "Rodada": 11 - }, - { - "ID": 190160, - "Rodada": 11 - }, - { - "ID": 190161, - "Rodada": 11 - }, - { - "ID": 190162, - "Rodada": 11 - }, - { - "ID": 190163, - "Rodada": 11 - }, - { - "ID": 190164, - "Rodada": 12 - }, - { - "ID": 190165, - "Rodada": 12 - }, - { - "ID": 190166, - "Rodada": 12 - }, - { - "ID": 190167, - "Rodada": 12 - }, - { - "ID": 190168, - "Rodada": 12 - }, - { - "ID": 190169, - "Rodada": 12 - }, - { - "ID": 190170, - "Rodada": 12 - }, - { - "ID": 190171, - "Rodada": 12 - }, - { - "ID": 190172, - "Rodada": 12 - }, - { - "ID": 190173, - "Rodada": 12 - }, - { - "ID": 190174, - "Rodada": 13 - }, - { - "ID": 190175, - "Rodada": 13 - }, - { - "ID": 190176, - "Rodada": 13 - }, - { - "ID": 190177, - "Rodada": 13 - }, - { - "ID": 190178, - "Rodada": 13 - }, - { - "ID": 190179, - "Rodada": 13 - }, - { - "ID": 190180, - "Rodada": 13 - }, - { - "ID": 190181, - "Rodada": 13 - }, - { - "ID": 190182, - "Rodada": 13 - }, - { - "ID": 190183, - "Rodada": 13 - }, - { - "ID": 190184, - "Rodada": 14 - }, - { - "ID": 190185, - "Rodada": 14 - }, - { - "ID": 190186, - "Rodada": 14 - }, - { - "ID": 190187, - "Rodada": 14 - }, - { - "ID": 190188, - "Rodada": 14 - }, - { - "ID": 190189, - "Rodada": 14 - }, - { - "ID": 190190, - "Rodada": 14 - }, - { - "ID": 190191, - "Rodada": 14 - }, - { - "ID": 190192, - "Rodada": 14 - }, - { - "ID": 190193, - "Rodada": 15 - }, - { - "ID": 190194, - "Rodada": 15 - }, - { - "ID": 190195, - "Rodada": 15 - }, - { - "ID": 190196, - "Rodada": 15 - }, - { - "ID": 190197, - "Rodada": 15 - }, - { - "ID": 190198, - "Rodada": 15 - }, - { - "ID": 190199, - "Rodada": 15 - }, - { - "ID": 190200, - "Rodada": 15 - }, - { - "ID": 190201, - "Rodada": 15 - }, - { - "ID": 190202, - "Rodada": 15 - }, - { - "ID": 190203, - "Rodada": 16 - }, - { - "ID": 190204, - "Rodada": 16 - }, - { - "ID": 190205, - "Rodada": 16 - }, - { - "ID": 190206, - "Rodada": 16 - }, - { - "ID": 190207, - "Rodada": 16 - }, - { - "ID": 190208, - "Rodada": 16 - }, - { - "ID": 190209, - "Rodada": 16 - }, - { - "ID": 190210, - "Rodada": 16 - }, - { - "ID": 190211, - "Rodada": 16 - }, - { - "ID": 190212, - "Rodada": 16 - }, - { - "ID": 190213, - "Rodada": 17 - }, - { - "ID": 190214, - "Rodada": 17 - }, - { - "ID": 190215, - "Rodada": 17 - }, - { - "ID": 190216, - "Rodada": 17 - }, - { - "ID": 190217, - "Rodada": 17 - }, - { - "ID": 190218, - "Rodada": 17 - }, - { - "ID": 190219, - "Rodada": 17 - }, - { - "ID": 190220, - "Rodada": 17 - }, - { - "ID": 190221, - "Rodada": 17 - }, - { - "ID": 190222, - "Rodada": 17 - }, - { - "ID": 190223, - "Rodada": 18 - }, - { - "ID": 190224, - "Rodada": 18 - }, - { - "ID": 190225, - "Rodada": 18 - }, - { - "ID": 190226, - "Rodada": 18 - }, - { - "ID": 190227, - "Rodada": 18 - }, - { - "ID": 190228, - "Rodada": 18 - }, - { - "ID": 190229, - "Rodada": 18 - }, - { - "ID": 190230, - "Rodada": 18 - }, - { - "ID": 190231, - "Rodada": 18 - }, - { - "ID": 190232, - "Rodada": 18 - }, - { - "ID": 190233, - "Rodada": 19 - }, - { - "ID": 190234, - "Rodada": 19 - }, - { - "ID": 190235, - "Rodada": 19 - }, - { - "ID": 190236, - "Rodada": 19 - }, - { - "ID": 190237, - "Rodada": 19 - }, - { - "ID": 190239, - "Rodada": 19 - }, - { - "ID": 190240, - "Rodada": 19 - }, - { - "ID": 190241, - "Rodada": 19 - }, - { - "ID": 190242, - "Rodada": 19 - }, - { - "ID": 190243, - "Rodada": 19 - }, - { - "ID": 190244, - "Rodada": 20 - }, - { - "ID": 190245, - "Rodada": 20 - }, - { - "ID": 190246, - "Rodada": 20 - }, - { - "ID": 190247, - "Rodada": 20 - }, - { - "ID": 190248, - "Rodada": 20 - }, - { - "ID": 190249, - "Rodada": 20 - }, - { - "ID": 190250, - "Rodada": 20 - }, - { - "ID": 190251, - "Rodada": 20 - }, - { - "ID": 190252, - "Rodada": 20 - }, - { - "ID": 190253, - "Rodada": 20 - }, - { - "ID": 190254, - "Rodada": 21 - }, - { - "ID": 190255, - "Rodada": 21 - }, - { - "ID": 190256, - "Rodada": 21 - }, - { - "ID": 190257, - "Rodada": 21 - }, - { - "ID": 190258, - "Rodada": 21 - }, - { - "ID": 190259, - "Rodada": 21 - }, - { - "ID": 190260, - "Rodada": 21 - }, - { - "ID": 190261, - "Rodada": 21 - }, - { - "ID": 190262, - "Rodada": 21 - }, - { - "ID": 190263, - "Rodada": 21 - }, - { - "ID": 190264, - "Rodada": 22 - }, - { - "ID": 190265, - "Rodada": 22 - }, - { - "ID": 190266, - "Rodada": 22 - }, - { - "ID": 190267, - "Rodada": 22 - }, - { - "ID": 190268, - "Rodada": 22 - }, - { - "ID": 190269, - "Rodada": 22 - }, - { - "ID": 190270, - "Rodada": 22 - }, - { - "ID": 190271, - "Rodada": 22 - }, - { - "ID": 190272, - "Rodada": 22 - }, - { - "ID": 190273, - "Rodada": 22 - }, - { - "ID": 190274, - "Rodada": 23 - }, - { - "ID": 190275, - "Rodada": 23 - }, - { - "ID": 190276, - "Rodada": 23 - }, - { - "ID": 190277, - "Rodada": 23 - }, - { - "ID": 190278, - "Rodada": 23 - }, - { - "ID": 190279, - "Rodada": 23 - }, - { - "ID": 190280, - "Rodada": 23 - }, - { - "ID": 190281, - "Rodada": 23 - }, - { - "ID": 190282, - "Rodada": 23 - }, - { - "ID": 190283, - "Rodada": 23 - }, - { - "ID": 190284, - "Rodada": 24 - }, - { - "ID": 190285, - "Rodada": 24 - }, - { - "ID": 190286, - "Rodada": 24 - }, - { - "ID": 190287, - "Rodada": 24 - }, - { - "ID": 190288, - "Rodada": 24 - }, - { - "ID": 190289, - "Rodada": 24 - }, - { - "ID": 190290, - "Rodada": 24 - }, - { - "ID": 190291, - "Rodada": 24 - }, - { - "ID": 190292, - "Rodada": 24 - }, - { - "ID": 190293, - "Rodada": 24 - }, - { - "ID": 190294, - "Rodada": 25 - }, - { - "ID": 190295, - "Rodada": 25 - }, - { - "ID": 190296, - "Rodada": 25 - }, - { - "ID": 190297, - "Rodada": 25 - }, - { - "ID": 190298, - "Rodada": 25 - }, - { - "ID": 190299, - "Rodada": 25 - }, - { - "ID": 190300, - "Rodada": 25 - }, - { - "ID": 190301, - "Rodada": 25 - }, - { - "ID": 190302, - "Rodada": 25 - }, - { - "ID": 190303, - "Rodada": 25 - }, - { - "ID": 190304, - "Rodada": 26 - }, - { - "ID": 190305, - "Rodada": 26 - }, - { - "ID": 190306, - "Rodada": 26 - }, - { - "ID": 190307, - "Rodada": 26 - }, - { - "ID": 190308, - "Rodada": 26 - }, - { - "ID": 190310, - "Rodada": 26 - }, - { - "ID": 190311, - "Rodada": 26 - }, - { - "ID": 190312, - "Rodada": 26 - }, - { - "ID": 190313, - "Rodada": 26 - }, - { - "ID": 190314, - "Rodada": 27 - }, - { - "ID": 190315, - "Rodada": 27 - }, - { - "ID": 190316, - "Rodada": 27 - }, - { - "ID": 190317, - "Rodada": 27 - }, - { - "ID": 190318, - "Rodada": 27 - }, - { - "ID": 190319, - "Rodada": 27 - }, - { - "ID": 190320, - "Rodada": 27 - }, - { - "ID": 190321, - "Rodada": 27 - }, - { - "ID": 190322, - "Rodada": 27 - }, - { - "ID": 190323, - "Rodada": 27 - }, - { - "ID": 190324, - "Rodada": 28 - }, - { - "ID": 190325, - "Rodada": 28 - }, - { - "ID": 190326, - "Rodada": 28 - }, - { - "ID": 190327, - "Rodada": 28 - }, - { - "ID": 190328, - "Rodada": 28 - }, - { - "ID": 190329, - "Rodada": 28 - }, - { - "ID": 190330, - "Rodada": 28 - }, - { - "ID": 190331, - "Rodada": 28 - }, - { - "ID": 190332, - "Rodada": 28 - }, - { - "ID": 190333, - "Rodada": 28 - }, - { - "ID": 190334, - "Rodada": 29 - }, - { - "ID": 190335, - "Rodada": 29 - }, - { - "ID": 190336, - "Rodada": 29 - }, - { - "ID": 190337, - "Rodada": 29 - }, - { - "ID": 190338, - "Rodada": 29 - }, - { - "ID": 190339, - "Rodada": 29 - }, - { - "ID": 190340, - "Rodada": 29 - }, - { - "ID": 190341, - "Rodada": 29 - }, - { - "ID": 190342, - "Rodada": 29 - }, - { - "ID": 190343, - "Rodada": 29 - }, - { - "ID": 190344, - "Rodada": 30 - }, - { - "ID": 190345, - "Rodada": 30 - }, - { - "ID": 190346, - "Rodada": 30 - }, - { - "ID": 190347, - "Rodada": 30 - }, - { - "ID": 190348, - "Rodada": 30 - }, - { - "ID": 190349, - "Rodada": 30 - }, - { - "ID": 190350, - "Rodada": 30 - }, - { - "ID": 190351, - "Rodada": 30 - }, - { - "ID": 190352, - "Rodada": 30 - }, - { - "ID": 190353, - "Rodada": 30 - }, - { - "ID": 190354, - "Rodada": 31 - }, - { - "ID": 190355, - "Rodada": 31 - }, - { - "ID": 190356, - "Rodada": 31 - }, - { - "ID": 190357, - "Rodada": 31 - }, - { - "ID": 190358, - "Rodada": 31 - }, - { - "ID": 190359, - "Rodada": 31 - }, - { - "ID": 190360, - "Rodada": 31 - }, - { - "ID": 190361, - "Rodada": 31 - }, - { - "ID": 190362, - "Rodada": 31 - }, - { - "ID": 190363, - "Rodada": 31 - }, - { - "ID": 190364, - "Rodada": 32 - }, - { - "ID": 190365, - "Rodada": 32 - }, - { - "ID": 190366, - "Rodada": 32 - }, - { - "ID": 190367, - "Rodada": 32 - }, - { - "ID": 190368, - "Rodada": 32 - }, - { - "ID": 190369, - "Rodada": 32 - }, - { - "ID": 190370, - "Rodada": 32 - }, - { - "ID": 190371, - "Rodada": 32 - }, - { - "ID": 190372, - "Rodada": 32 - }, - { - "ID": 190373, - "Rodada": 32 - }, - { - "ID": 190374, - "Rodada": 33 - }, - { - "ID": 190375, - "Rodada": 33 - }, - { - "ID": 190376, - "Rodada": 33 - }, - { - "ID": 190377, - "Rodada": 33 - }, - { - "ID": 190378, - "Rodada": 33 - }, - { - "ID": 190379, - "Rodada": 33 - }, - { - "ID": 190380, - "Rodada": 33 - }, - { - "ID": 190381, - "Rodada": 33 - }, - { - "ID": 190382, - "Rodada": 33 - }, - { - "ID": 190383, - "Rodada": 33 - }, - { - "ID": 190384, - "Rodada": 34 - }, - { - "ID": 190385, - "Rodada": 34 - }, - { - "ID": 190386, - "Rodada": 34 - }, - { - "ID": 190387, - "Rodada": 34 - }, - { - "ID": 190388, - "Rodada": 34 - }, - { - "ID": 190389, - "Rodada": 34 - }, - { - "ID": 190390, - "Rodada": 34 - }, - { - "ID": 190391, - "Rodada": 34 - }, - { - "ID": 190392, - "Rodada": 34 - }, - { - "ID": 190393, - "Rodada": 34 - }, - { - "ID": 190394, - "Rodada": 35 - }, - { - "ID": 190395, - "Rodada": 35 - }, - { - "ID": 190396, - "Rodada": 35 - }, - { - "ID": 190397, - "Rodada": 35 - }, - { - "ID": 190398, - "Rodada": 35 - }, - { - "ID": 190399, - "Rodada": 35 - }, - { - "ID": 190400, - "Rodada": 35 - }, - { - "ID": 190401, - "Rodada": 35 - }, - { - "ID": 190402, - "Rodada": 35 - }, - { - "ID": 190403, - "Rodada": 35 - }, - { - "ID": 190404, - "Rodada": 36 - }, - { - "ID": 190405, - "Rodada": 36 - }, - { - "ID": 190406, - "Rodada": 36 - }, - { - "ID": 190407, - "Rodada": 36 - }, - { - "ID": 190408, - "Rodada": 36 - }, - { - "ID": 190409, - "Rodada": 36 - }, - { - "ID": 190410, - "Rodada": 36 - }, - { - "ID": 190411, - "Rodada": 36 - }, - { - "ID": 190412, - "Rodada": 36 - }, - { - "ID": 190413, - "Rodada": 36 - }, - { - "ID": 190414, - "Rodada": 37 - }, - { - "ID": 190415, - "Rodada": 37 - }, - { - "ID": 190416, - "Rodada": 37 - }, - { - "ID": 190417, - "Rodada": 37 - }, - { - "ID": 190418, - "Rodada": 37 - }, - { - "ID": 190419, - "Rodada": 37 - }, - { - "ID": 190420, - "Rodada": 37 - }, - { - "ID": 190421, - "Rodada": 37 - }, - { - "ID": 190422, - "Rodada": 37 - }, - { - "ID": 190423, - "Rodada": 37 - }, - { - "ID": 190424, - "Rodada": 38 - }, - { - "ID": 190425, - "Rodada": 38 - }, - { - "ID": 190426, - "Rodada": 38 - }, - { - "ID": 190427, - "Rodada": 38 - }, - { - "ID": 190428, - "Rodada": 38 - }, - { - "ID": 190429, - "Rodada": 38 - }, - { - "ID": 190430, - "Rodada": 38 - }, - { - "ID": 190431, - "Rodada": 38 - }, - { - "ID": 190432, - "Rodada": 38 - }, - { - "ID": 190436, - "Rodada": 14 - }, - { - "ID": 190437, - "Rodada": 26 - }, - { - "ID": 191150, - "Rodada": 38 - } -] \ No newline at end of file diff --git a/README.md b/README.md index dfdc553..9ad130e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ -Este é um scraper que roda na plataforma [Morph](https://morph.io). +# CartolaFCScraper -[Download Dados](https://morph.io/thevtm/CartolaFCScraper) \ No newline at end of file +Este é um scraper para o CartolaFC que roda na plataforma [Morph](https://morph.io). +O scraper apenas armazena os dados obtidos do CartolaFC, nenhuma modificação é feita. + +[Pagina do CartolaFCScraper no Morph.](https://morph.io/thevtm/CartolaFCScraper) + +[Dados das verções anteriores.](https://github.com/thevtm/CartolaFCDados) + +## API + + https://api.cartolafc.globo.com/mercado/status + https://api.cartolafc.globo.com/atletas/mercado + https://api.cartolafc.globo.com/atletas/pontuados + https://api.cartolafc.globo.com/partidas + https://api.cartolafc.globo.com/clubes diff --git a/Scrapers/Lances.py b/Scrapers/Lances.py deleted file mode 100644 index 8e97b4a..0000000 --- a/Scrapers/Lances.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import re -import itertools as it -import json -from lxml import html -import scraperwiki -from LancesScrapers import Substituicoes -from LancesScrapers import Cartoes -from LancesScrapers import Gols - -def ScrapeLances(Rodada, USER_AGENT): - - # Consts - PARTIDAS_RODADA_URL = 'http://globoesporte.globo.com/servico/esportes_campeonato/responsivo/widget-uuid/09021843-e53d-4020-80f7-302a15756585/fases/fase-unica-brasileiro-2015/rodada/{0}/jogos.html' - LANCES_URL = 'mensagens.json' - - - # Fetch links das Partidas - - print '[LOG] Obtendo os links dos Lances' - - PartidasRodadaLink = PARTIDAS_RODADA_URL.format(Rodada) - PartidasRodadaLinkData = scraperwiki.scrape(PartidasRodadaLink, user_agent=USER_AGENT) - PartidasLinks = html.fromstring(PartidasRodadaLinkData).xpath('//a/@href') - - print '[LOG] Links das Lances obtidos' - - - # Download dados dos lances - - print '[LOG] Download Lances Iniciado' - - LancesJSON = [] - for i, l in enumerate(PartidasLinks): - url = l + '/' + LANCES_URL - print '[LOG] Baixando ', i + 1, '/', len(PartidasLinks) - data = scraperwiki.scrape(url) - jData = json.loads(data) - ## Adiciona PartidaID em cada lance - PartidaID = int(re.search('"jogo_id": (\d+),', data).groups()[0]) - for j in jData: - j['PartidaID'] = PartidaID - LancesJSON.append(jData) - - # Concatena Lances - LancesJSON = list(it.chain.from_iterable(LancesJSON)) - - print '[LOG] Download Lances Terminado' - - - # Minera dados - print '[LOG] Processamento de Lances Iniciado' - - # Minera Substituicoes - Substituicoes.ScrapeSubstituicoes(LancesJSON) - - # Minera Cartões - Cartoes.ScrapeCartoes(LancesJSON) - - # Minera Gols - Gols.ScrapeGols(LancesJSON) - - print '[LOG] Processamento de Lances Terminado' - diff --git a/Scrapers/LancesScrapers/Cartoes.py b/Scrapers/LancesScrapers/Cartoes.py deleted file mode 100644 index 20ee460..0000000 --- a/Scrapers/LancesScrapers/Cartoes.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import scraperwiki - - -def ScrapeCartoes(LancesJSON): - print '[LOG] Processamento de Lances.Cartões Iniciado' - - def isCartao(x): - try: - return (x['tipo'] == 'LANCE_CARTAO' and 'cartao' in x) - except: - return False - - # Filtra Cartões - CartoesJSON = filter(isCartao, LancesJSON) - - CartoesDict = [] - for c in CartoesJSON: - cDict = {} - cDict['ID'] = c['id'] - cDict['PartidaID'] = c['PartidaID'] - cDict['Periodo'] = c['periodo_sigla'] - cDict['Momento'] = c['momento'] - cDict['Time'] = c['nome_time'] - cDict['Apelido'] = c['cartao']['nome_jogador'] - cDict['AtletaID'] = c['cartao']['jogador_id'] - cDict['Cartao'] = c['cartao']['tipo'] - - # Minera ID do clube - if(c['jogo']['equipe_mandante']['nome'] == c['nome_time']): - cDict['TimeID'] = c['jogo']['equipe_mandante']['id'] - else: - cDict['TimeID'] = c['jogo']['equipe_visitante']['id'] - - CartoesDict.append(cDict) - - print '[LOG] Processamento de Lances.Cartões Terminado' - - - # Save DataFrame to SQLite - - print '[LOG] Transferindo Lances.Cartões para SQLite' - - scraperwiki.sqlite.save(unique_keys=['ID'], - data=CartoesDict, - table_name='Cartoes') - - print '[LOG] Lances.Cartões Salvas' diff --git a/Scrapers/LancesScrapers/Gols.py b/Scrapers/LancesScrapers/Gols.py deleted file mode 100644 index 376c0cb..0000000 --- a/Scrapers/LancesScrapers/Gols.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import scraperwiki - - -def ScrapeGols(LancesJSON): - print '[LOG] Processamento de Lances.Gols Iniciado' - - def isGol(x): - try: - return (x['tipo'] == 'LANCE_GOL' and 'gol' in x and 'momento' in x) - except: - return False - - # Filtra Gols - GolsJSON = filter(isGol, LancesJSON) - - GolsDict = [] - for g in GolsJSON: - gDict = {} - gDict['ID'] = g['id'] - gDict['PartidaID'] = g['PartidaID'] - gDict['Periodo'] = g['periodo_sigla'] - gDict['Momento'] = g['momento'] - gDict['Time'] = g['nome_time'] - gDict['Apelido'] = g['gol']['autor'] - gDict['AtletaID'] = g['gol']['autor_id'] - gDict['Contra'] = g['gol']['contra'] - - # Minera ID do clube - if(g['jogo']['equipe_mandante']['nome'] == g['nome_time']): - gDict['TimeID'] = g['jogo']['equipe_mandante']['id'] - else: - gDict['TimeID'] = g['jogo']['equipe_visitante']['id'] - - GolsDict.append(gDict) - - print '[LOG] Processamento de Lances.Gols Terminado' - - # Save DataFrame to SQLite - - print '[LOG] Transferindo Lances.Gols para SQLite' - - scraperwiki.sqlite.save(unique_keys=['ID'], - data=GolsDict, - table_name='Gols') - - print '[LOG] Lances.Gols Salvas' diff --git a/Scrapers/LancesScrapers/Substituicoes.py b/Scrapers/LancesScrapers/Substituicoes.py deleted file mode 100644 index bc526bd..0000000 --- a/Scrapers/LancesScrapers/Substituicoes.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import scraperwiki - -def ScrapeSubstituicoes(LancesJSON): - print '[LOG] Processamento de Lances.Substituições Iniciado' - - def isSubstituicao(x): - try: - return (x['tipo'] == 'LANCE_SUBSTITUICAO' and 'substituicao' in x) - except: - return False - - - # Filtra Substituições - SubstituicoesJSON = filter(isSubstituicao, LancesJSON) - - SubstituicoesDict = [] - for s in SubstituicoesJSON: - sDict = {} - sDict['ID'] = s['id'] - sDict['PartidaID'] = s['PartidaID'] - sDict['Periodo'] = s['periodo_sigla'] - sDict['Momento'] = s['momento'] - sDict['Time'] = s['nome_time'] - sDict['TimeID'] = s['substituicao']['time_id'] - sDict['SaiuApelido'] = s['substituicao']['nome'] - sDict['SaiuAtletaID'] = s['substituicao']['pessoa_id'] - sDict['SaiuTitular'] = s['substituicao']['titular'] - sDict['SaiuPosicao'] = s['substituicao']['posicao'] - sDict['EntrouApelido'] = s['substituicao']['substituido_por']['nome'] - sDict['EntrouAtletaID'] = s['substituicao']['substituido_por']['pessoa_id'] - sDict['EntrouTitular'] = s['substituicao']['substituido_por']['titular'] - sDict['EntrouPosicao'] = s['substituicao']['substituido_por']['posicao'] - SubstituicoesDict.append(sDict) - - print '[LOG] Processamento de Lances.Substituições Terminado' - - # Save DataFrame to SQLite - - print '[LOG] Transferindo Lances.Substituições para SQLite' - - scraperwiki.sqlite.save(unique_keys=['ID'], - data=SubstituicoesDict, - table_name='Substituicoes') - - print '[LOG] Lances.Substituições Salvas' - diff --git a/Scrapers/LancesScrapers/__init__.py b/Scrapers/LancesScrapers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Scrapers/Partidas.py b/Scrapers/Partidas.py deleted file mode 100644 index 569c931..0000000 --- a/Scrapers/Partidas.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import json -import scraperwiki - - -def ScrapePartidas(PartidasID, USER_AGENT): - - # Consts - FICHA_URL = 'http://globoesporte.globo.com/dynamo/confronto/jogo/{0}/ficha.json' - - # Download dados das partidas - - print '[LOG] Downloading Partidas Iniciado' - - PartidasJSON = [] - for i, p, in enumerate(PartidasID): - url = FICHA_URL.format(p) - - print '[LOG] Baixando Partida', i + 1, '/', len(PartidasID) - PartidasJSON.append(scraperwiki.scrape(url, user_agent=USER_AGENT)) - - print '[LOG] Downloading Partidas Terminado' - - # Minera dados - - print '[LOG] Processamento de Partidas Iniciado' - - PartidasJSON = [json.loads(j) for j in PartidasJSON] - PartidasDict = [] - - for p in PartidasJSON: - pDict = {} - pDict['ID'] = p['jogo_id'] - pDict['Rodada'] = p['rodada'] - pDict['EquipeMandante'] = p['equipe_mandante']['slug'] - pDict['EquipeVisitante'] = p['equipe_visitante']['slug'] - pDict['PlacarMandante'] = p['placar_mandante'] - pDict['PlacarVisitante'] = p['placar_visitante'] - - PartidasDict.append(pDict) - - print '[LOG] Processamento de Partidas Terminado' - - # Save DataFrame to SQLite - - print '[LOG] Transferindo Partidas para SQLite' - - scraperwiki.sqlite.save(unique_keys=['ID'], - data=PartidasDict, - table_name='Partidas') - - print '[LOG] Partidas Salvas' diff --git a/Scrapers/Scouts.py b/Scrapers/Scouts.py deleted file mode 100644 index 6ea9dd5..0000000 --- a/Scrapers/Scouts.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import itertools as it -import json -import scraperwiki -import mechanize -import cookielib - -def DownloadScouts(LOGIN_EMAIL, LOGIN_SENHA, USER_AGENT): - # Consts - SCOUTS_URL = 'http://cartolafc.globo.com/mercado/filtrar.json?page=' - LOGIN_URL = 'https://loginfree.globo.com/login/438' - - # Cria o mechanize browser - print '[LOG] Downloading Scouts Iniciado' - - # Inicialisa browser - br = mechanize.Browser() - - # Cookie Jar - cj = cookielib.LWPCookieJar() - br.set_cookiejar(cj) - - # Define user-agent - br.addheaders = [('User-agent', USER_AGENT)] - - # Loga no CartolaFC - - # Abre a página de login Cartola através do mechanize - br.open(LOGIN_URL) - - # Na página, selecionamos o primeiro form presente. 'nr=0' indica que estamos - # selecionando o form de índice 0 dentre os encontrados. - # Analisando a página, vemos que realmente só há um form. - # Após selecionarmos o form, preenchemos os campos com username e - # senha que permitam fazer o login. - - br.select_form(nr=0) - br.form['login-passaporte'] = LOGIN_EMAIL ## CartolaFC Login - br.form['senha-passaporte'] = LOGIN_SENHA ## CartolaFC Senha - br.submit() - - # Download Scouts - - jsonRaw = [] - for i in it.count(1): - url = SCOUTS_URL + str(i) - - r = br.open(url).read() - j = json.loads(r) - jsonRaw.append(r) - - pgAtual = int(j['page']['atual']) - pgTotal = int(j['page']['total']) - - print '[LOG] Baixando ', i, '/', pgTotal - - if pgAtual == pgTotal: - break - - print '[LOG] Downloading Scouts Terminado' - - return jsonRaw - - -def ProcessScouts(jsonRaw): - print '[LOG] Processamento de Scouts Iniciado' - - # Minera Rodada - rodada = jsonRaw[0]['rodada_id'] - 1 - - # Concatena lista de atletas dos arquivos - atletasJSON = [j['atleta'] for j in jsonRaw] - atletasJSON = list(it.chain(*atletasJSON)) - - # Minera Scouts - # e concatena em um DataFrame - - ScoutsDict = [] - - for atleta in atletasJSON: - scoutDict = {s['nome']: s['quantidade'] for s in atleta['scout']} # Add Scouts - scoutDict['Rodada'] = rodada - scoutDict['Atleta'] = atleta['id'] - scoutDict['Apelido'] = atleta['apelido'] - scoutDict['Clube'] = atleta['clube']['slug'] - scoutDict['ClubeID'] = atleta['clube']['id'] - scoutDict['Posicao'] = atleta['posicao']['abreviacao'] - scoutDict['PosicaoID'] = atleta['posicao']['id'] - scoutDict['Status'] = atleta['status'] - scoutDict['Pontos'] = float(atleta['pontos']) - scoutDict['PontosMedia'] = float(atleta['media']) - scoutDict['Preco'] = float(atleta['preco']) - scoutDict['PrecoVariacao'] = float(atleta['variacao']) - scoutDict['Mando'] = atleta['clube']['id'] == atleta['partida_clube_casa']['id'] - scoutDict['Jogos'] = atleta['jogos'] - scoutDict['PartidaCasa'] = atleta['partida_clube_casa']['slug'] - scoutDict['PartidaCasaID'] = atleta['partida_clube_casa']['id'] - scoutDict['PartidaVisitante'] = atleta['partida_clube_visitante']['slug'] - scoutDict['PartidaVisitanteID'] = atleta['partida_clube_visitante']['id'] - scoutDict['PartidaData'] = atleta['partida_data'] - - ScoutsDict.append(scoutDict) - - print '[LOG] Processamento de Scouts Terminado' - - return ScoutsDict - - -def ScrapeScouts(LOGIN_EMAIL, LOGIN_SENHA, USER_AGENT): - - # Download dados - jsonRaw = DownloadScouts(LOGIN_EMAIL, LOGIN_SENHA, USER_AGENT) - - # Processa dados - jsonRaw = [json.loads(j) for j in jsonRaw] - ScoutsDict = ProcessScouts(jsonRaw) - - # Save DataFrame to SQLite - - print '[LOG] Transferindo Scouts para SQLite' - - scraperwiki.sqlite.save(unique_keys=['Atleta', 'Rodada'], - data=ScoutsDict, - table_name='data') - - print '[LOG] Scouts Salvos' diff --git a/Scrapers/__init__.py b/Scrapers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/package.json b/package.json new file mode 100644 index 0000000..db2471e --- /dev/null +++ b/package.json @@ -0,0 +1,30 @@ +{ + "name": "cartola-fc-scraper", + "version": "0.0.1", + "description": "Este é um scraper que roda na plataforma [Morph](https://morph.io).", + "main": "scraper.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/thevtm/CartolaFCScraper.git" + }, + "keywords": [ + "CartolaFC", + "Scraper" + ], + "author": "Vinícius Tabille Manjabosco", + "license": "MIT", + "bugs": { + "url": "https://github.com/thevtm/CartolaFCScraper/issues" + }, + "homepage": "https://github.com/thevtm/CartolaFCScraper#readme", + "engines": { + "node": "6.2.0" + }, + "dependencies": { + "request-promise": "^3.0.0", + "sqlite3": "^3.1.4" + } +} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9b7d9e2..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Install custom version of scraperwiki library --e git+http://github.com/openaustralia/scraperwiki-python.git@morph_defaults#egg=scraperwiki - -# Libs -mechanize==0.2.5 -lxml==3.4.4 diff --git a/scraper.js b/scraper.js new file mode 100644 index 0000000..d0b4fec --- /dev/null +++ b/scraper.js @@ -0,0 +1,61 @@ +let Promise = require("bluebird"); +let rp = require('request-promise'); +let sqlite3 = require("sqlite3").verbose(); + +/* CONSTANTS */ + +const API_MERCADO_STATUS = 'https://api.cartolafc.globo.com/mercado/status' +const API_ATLETAS_MERCADO = 'https://api.cartolafc.globo.com/atletas/mercado' +const API_ATLETAS_PONTUADOS = 'https://api.cartolafc.globo.com/atletas/pontuados' +const API_PARTIDAS = 'https://api.cartolafc.globo.com/partidas' +const API_CLUBES = 'https://api.cartolafc.globo.com/clubes' + + +/* REQUEST FILES */ +console.log('Requisitando os arquivos...') + +let requests_datetime = new Date(); + +let requests = Promise.join( + rp(API_MERCADO_STATUS), + rp(API_ATLETAS_MERCADO), + rp(API_ATLETAS_PONTUADOS), + rp(API_PARTIDAS), + rp(API_CLUBES) +) + +/* STORE FILES */ +requests.spread( + (mercadoStatus, atletasMercado, atletasPontuados, partidas, clubes) => { + + console.log('Arquivos requisitados com sucesso.') + + // Open a database handle + let db = new sqlite3.Database("data.sqlite"); + db.serialize( + () => { + + console.log('Armazenando dados no banco de dados...') + + // Create new table + db.run(`CREATE TABLE IF NOT EXISTS data + ( + DataHora TEXT, + API_MERCADO_STATUS TEXT, + API_ATLETAS_MERCADO TEXT, + API_ATLETAS_PONTUADOS TEXT, + API_PARTIDAS TEXT, + API_CLUBES TEXT + )`); + + // Insert a new record + let statement = db.prepare("INSERT INTO data VALUES (?, ?, ?, ?, ?, ?)"); + statement.run(requests_datetime.toISOString(), mercadoStatus, + atletasMercado, atletasPontuados, partidas, clubes); + statement.finalize(); + + console.log('Dados armazenados no banco de dados com sucesso.') + } + ) + } +) diff --git a/scraper.py b/scraper.py deleted file mode 100644 index 0151d2f..0000000 --- a/scraper.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -## -# Copyright (c) 2015 Vinícius Tabille Manjabosco -## -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -## -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -## -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import os -import json -import scraperwiki -from Scrapers import Scouts -from Scrapers import Partidas -from Scrapers import Lances - -# Consts -LOGIN_EMAIL = os.environ['MORPH_LOGIN_EMAIL'] ## CartolaFC Login -LOGIN_SENHA = os.environ['MORPH_LOGIN_SENHA'] ## CartolaFC Senha -USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' -PARTIDAS_DATA = 'Data/Partidas.json' - -# Functions -def FetchUltimaRodadaDosScouts(): - try: - Rodada = scraperwiki.sqlite.select("max(Rodada) as Rodada from data")[0]['Rodada'] - except Exception, e: - DEFAULT_RODADA_ERROR = 0 - print '[ERROR] Nao foi possivel obter a ultima rodada da tabela Scouts, usando rodada:', DEFAULT_RODADA_ERROR - Rodada = DEFAULT_RODADA_ERROR - else: - print '[LOG] Ultima rodada na tabela Scouts:', Rodada - finally: - return Rodada - - -# Carrega ids das partidas -with open(PARTIDAS_DATA) as f: - PartidasData = json.load(f) - -# Scrape Scouts -Scouts.ScrapeScouts(LOGIN_EMAIL, LOGIN_SENHA, USER_AGENT) - -# Obtem a ultima rodada da tabela Scouts -ultimaRodada = FetchUltimaRodadaDosScouts() - -# Scrape Partidas -PartidasID = [p['ID'] for p in PartidasData if p['Rodada'] == ultimaRodada] -Partidas.ScrapePartidas(PartidasID, USER_AGENT) - -# Scrape Lances -Lances.ScrapeLances(ultimaRodada, USER_AGENT)