add unique IPs and users + count

.gitattributes

@@ -1,4 +1,2 @@
-src/IPs.log filter=lfs diff=lfs merge=lfs -text
-src/users.log filter=lfs diff=lfs merge=lfs -text
-archives/users.log.tar.gz filter=lfs diff=lfs merge=lfs -text
-archives/IPs.log.tar.gz filter=lfs diff=lfs merge=lfs -text
+src/ filter=lfs diff=lfs merge=lfs -text
+archives/ filter=lfs diff=lfs merge=lfs -text

archive.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+tar -czvf archives/IPs.log.tar.gz src/IPs.log
+tar -czvf archives/unique.IPs.log.tar.gz src/unique.IPs.log
+tar -czvf archives/unique.IPs.count.log.tar.gz src/unique.IPs.count.log
+
+tar -czvf archives/users.log.tar.gz src/users.log
+tar -czvf archives/unique.users.log.tar.gz src/unique.users.log
+tar -czvf archives/unique.users.count.log.tar.gz src/unique.users.count.log

dedupe.py

@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+
+IPs = open("src/IPs.log").readlines()
+uniqueIPs = set(IPs)
+open("src/unique.IPs.log", "w").writelines(uniqueIPs)
+users = open("src/users.log").readlines()
+uniqueUsers = set(users)
+open("src/unique.users.log", "w").writelines(uniqueUsers)

order.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+python3 ./dedupe.py
+
+rm src/unique.IPs.count.log
+rm src/unique.users.count.log
+
+while IFS= read -r line; do count=$(grep -c "$line" src/IPs.log); echo $line:$count >> src/unique.IPs.count.log; done < src/unique.IPs.log
+
+while IFS= read -r line; do count=$(grep -Pc "\A$line$" src/users.log); echo $line:$count >> src/unique.users.count.log; done < src/unique.users.log
+
+python3 ./sort.py
+
+echo DONE

sort.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+from operator import itemgetter
+
+IPs = open("src/unique.IPs.count.log").readlines()
+
+IPcounts = []
+for IP in IPs:
+  ip, count = IP.split(":")
+  IPcounts.append((ip, int(count)))
+
+with open("src/unique.IPs.count.log", "w") as IPsfile:
+  for ip, count in sorted(IPcounts, key=itemgetter(1), reverse=True):
+    IPsfile.write("{}:{}\n".format(ip, count))
+
+users = open("src/unique.users.count.log").readlines()
+
+Usercounts = []
+for user in users:
+  u, count = user.split(":")
+  Usercounts.append((u, int(count)))
+
+with open("src/unique.users.count.log", "w") as Usersfile:
+  for u, count in sorted(Usercounts, key=itemgetter(1), reverse=True):
+    Usersfile.write("{}:{}\n".format(u, count))