Code for populating table added

lifeiitkbackend/settings.py

@@ -9,12 +9,24 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/2.2/ref/settings/
 """
-
+import yaml
 import os
-
+from django.core.exceptions import ImproperlyConfigured
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+with open(os.path.join(BASE_DIR, 'secrets.yml')) as secrets_file:
+    secrets = yaml.safe_load(secrets_file)
+
+def get_secret(setting, secrets = secrets):
+    try:
+        return secrets[setting]
+    except KeyError:
+        raise ImproperlyConfigured("Set the {} setting".format(setting))
+
+
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
@@ -38,6 +50,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'users',
+    'django_extensions',
 ]
 
 MIDDLEWARE = [
@@ -76,8 +89,11 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'students',
+        'USER': 'aditya',
+        'PASSWORD': get_secret('DB_PASSWORD'),
+        'HOST': 'localhost', 
     }
 }
 

requirements.txt

@@ -1 +1,5 @@
 django
+re
+psycopg2
+requests
+bs4

secrets.yml

@@ -0,0 +1 @@
+"DB_PASSWORD": "password" 

users/jobs/daily/scrape.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+from django_extensions.management.jobs import DailyJob
+
+from bs4 import BeautifulSoup
+import re
+import requests
+import psycopg2
+
+conn = psycopg2.connect(host="localhost", database="students",user="aditya",password="password")
+c = conn.cursor()
+s = requests.Session()
+s.get("https://oa.cc.iitk.ac.in/Oa/Jsp/Main_Frameset.jsp")
+s.get("https://oa.cc.iitk.ac.in/Oa/Jsp/Main_Intro.jsp?frm='SRCH'")
+s.get("https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITK_Srch.jsp?typ=stud")
+
+headers = {
+    "Referer": "https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITK_Srch.jsp?typ=stud"
+}
+
+headers1 = {
+    "Referer": "https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITk_SrchStudRoll_new.jsp"
+}
+
+payload = {
+    'k4': 'oa',
+    'numtxt': '',
+    'recpos': 0,
+    'str': '',
+    'selstudrol': '',
+    'selstuddep': '',
+    'selstudnam': '',
+    'txrollno': '',
+    'Dept_Stud': '',
+    'selnam1': '',
+    'mail': ''
+}
+
+payload1 = {
+    'typ': ['stud'] * 12,
+    'numtxt': '',
+    'sbm': ['Y'] * 12
+}
+TOTAL = 8385
+r = s.post("https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITk_SrchStudRoll_new.jsp", headers=headers, data=payload)
+soup = BeautifulSoup(r.text, 'html.parser')
+
+class Job(DailyJob):
+
+    def process_response_soup(self,soup, c):
+
+
+
+
+        for link in soup.select('.TableText a'):
+            roll = link.get_text().strip()
+            payload1['numtxt'] = roll
+            r1 = s.post("https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITk_SrchRes_new.jsp", headers=headers1, data=payload1)
+            soup1 = BeautifulSoup(r1.text, 'html.parser')
+
+
+
+            name = ''
+            program = ''
+            dept = ''
+            hall = ''
+            room = ''
+            username = ''
+            blood_group = ''
+            gender = ''
+            hometown = ''
+
+            for para in soup1.select('.TableContent p'):
+                body = para.get_text().strip()
+                field = body.split(':')
+                key = field[0].strip()
+                value = field[1].strip()
+                if key == 'Name':
+                    name = value.lower().title()
+                elif key == 'Program':
+                    program = value
+                elif key == 'Department':
+                    dept = value.lower().title()
+                elif key == 'Hostel Info':
+                    if len(value.split(',')) > 1:
+                        hall = value.split(',')[0].strip()
+                        room = value.split(',')[1].strip()
+                elif key == 'E-Mail':
+                    if len(value.split('@')) > 1:
+                        username = value.split('@')[0].strip()
+                elif key == 'Blood Group':
+                    blood_group = value
+                elif key == 'Gender':
+                    if len(value.split('\t')) > 1:
+                        gender = value.split('\t')[0].strip()
+                else:
+                    print("{} {}".format(key, value))
+
+            body = soup1.prettify()
+            if len(body.split('Permanent Address :')) > 1:
+                address = body.split('Permanent Address :')[1].split(',')
+                if len(address) > 2:
+                    address = address[len(address) - 3: len(address) - 1]
+                    hometown = "{}, {}".format(address[0], address[1])
+                    image = 'http://oa.cc.iitk.ac.in/Oa/Jsp/Photo/' + str(roll) + '_0.jpg'
+                    c.execute("INSERT INTO users_user (roll, username,image,name,program,dept,hall,room,blood_group,gender,hometown) VALUES ("+roll+", '"+username+"','"+image+"', '"+name+"', '"+program+"', '"+dept+"', '"+hall+"', '"+room+"', '"+blood_group+"', '"+gender+"', '"+hometown+"') ON CONFLICT(roll) DO UPDATE SET name=Excluded.name, image=Excluded.image, username=Excluded.username, program=Excluded.program, dept=Excluded.dept, hall=Excluded.hall, blood_group=Excluded.blood_group,gender=Excluded.gender,hometown=Excluded.hometown,room=Excluded.room")
+                    #c.execute("INSERT INTO users_user (user_img ) VALUES" ('http://oa.cc.iitk.ac.in/Oa/Jsp/Photo/' + str(roll) + '_0.jpg'))
+
+
+    def execute(self,soup,c):
+        for link in soup.select('.DivContent'):
+            substituted = re.sub(r'\s+', ' ', link.text)
+            pattern = re.compile(r'\s*You are viewing 1 to 12 records out of (\d+) records\s*')
+            match = pattern.match(substituted)
+            TOTAL = int(match.group(1))
+            print("Total: {}".format(TOTAL))
+        self.process_response_soup(soup, c)
+
+
+
+        for i in range(12, TOTAL+1, 12):
+            payload['recpos'] = i
+            r = s.post("https://oa.cc.iitk.ac.in/Oa/Jsp/OAServices/IITk_SrchStudRoll_new.jsp", headers=headers, data=payload)
+            soup = BeautifulSoup(r.text, 'html.parser')
+            self.process_response_soup(soup, c)
+
+
+
+
+
+            print("Processed {}".format(i + 12))
+            conn.commit()
+        conn.close()
+
+x = Job()
+x.execute(soup,c)

users/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# Generated by Django 2.2.1 on 2019-06-02 14:51
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='User',
+            fields=[
+                ('roll', models.CharField(max_length=10, primary_key=True, serialize=False, unique=True)),
+                ('username', models.CharField(max_length=10)),
+                ('name', models.CharField(max_length=100)),
+                ('program', models.CharField(max_length=20)),
+                ('dept', models.CharField(max_length=50)),
+                ('hall', models.CharField(max_length=15)),
+                ('room', models.CharField(max_length=20)),
+                ('blood_group', models.CharField(max_length=4)),
+                ('gender', models.CharField(max_length=10)),
+                ('hometown', models.CharField(max_length=100)),
+                ('fblink', models.URLField(max_length=120)),
+                ('por', django.contrib.postgres.fields.jsonb.JSONField()),
+                ('earlier_login', models.BooleanField()),
+            ],
+        ),
+    ]

users/migrations/0002_auto_20190602_1859.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.1 on 2019-06-02 18:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='earlier_login',
+            field=models.BooleanField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='fblink',
+            field=models.URLField(max_length=120, null=True),
+        ),
+    ]

users/migrations/0003_auto_20190602_1901.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.1 on 2019-06-02 19:01
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0002_auto_20190602_1859'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='por',
+            field=django.contrib.postgres.fields.jsonb.JSONField(null=True),
+        ),
+    ]

users/migrations/0004_auto_20190602_1920.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.1 on 2019-06-02 19:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0003_auto_20190602_1901'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='blood_group',
+            field=models.CharField(max_length=14),
+        ),
+    ]

users/migrations/0005_auto_20190603_0816.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.1 on 2019-06-03 08:16
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0004_auto_20190602_1920'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='earlier_login',
+            field=models.BooleanField(default=0, null=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='por',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default={}, null=True),
+        ),
+    ]

users/migrations/0006_auto_20190603_0821.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.1 on 2019-06-03 08:21
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0005_auto_20190603_0816'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='por',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True),
+        ),
+    ]

users/migrations/0007_user_user_img.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.1 on 2019-06-03 11:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0006_auto_20190603_0821'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='user',
+            name='user_img',
+            field=models.ImageField(blank=True, null=True, upload_to=''),
+        ),
+    ]

users/migrations/0008_auto_20190603_1249.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.1 on 2019-06-03 12:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0007_user_user_img'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='roll',
+            field=models.CharField(max_length=20, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='username',
+            field=models.CharField(max_length=20),
+        ),
+    ]

users/migrations/0009_auto_20190603_1834.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.1 on 2019-06-03 18:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0008_auto_20190603_1249'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='user',
+            old_name='user_img',
+            new_name='image',
+        ),
+    ]