source.py

# coding: utf-8

# In[1]:


from bs4 import BeautifulSoup  # for finding relevant word
import os
from nltk import PorterStemmer
import operator  # for itemgetter
import math  # for log

# In[2]:


termIDs = {}
docIDs = {}
termDF = {}
Queries = {}
# docIND = {}
tDocTerms = {}  # total terms in a document

# In[3]:


Stop_word_file = open("F:\\PycharmProjects\\SearchEngine\\files\\stoplist.txt", 'r', encoding='utf8', errors='ignore')
stopwords = Stop_word_file.readlines()
stopwords = [x.strip() for x in stopwords]  # removes all leading and trailing spaces from string
Stop_word_file.close()


# In[4]:


def processQuery():
    for key, val in Queries.items():
        sentence = Queries[key]
        words = sentence.lower().split()
        # REGEXXXXXXXXXXXX
        symbols = r"!@#$%^&*”“…’‘|~`()_-\"+=/\}{[,].<';:>?"  # use \ before character that are like eof
        for i in range(0, len(symbols)):
            words = [word.replace(symbols[i], "") for word in words]

        temp = []
        for index, w in enumerate(words):
            if w in stopwords:
                temp.append(w)
            else:
                stemmed = PorterStemmer().stem(w)
                words[index] = stemmed

        for t in temp:
            words.remove(t)

        Queries[key] = words


# In[5]:


def parseQueryTopics():
    QT = open("F:\\PycharmProjects\\SearchEngine\\files\\p2\\topics.xml", mode="r", encoding="utf-8", errors='ignore')
    source_code = QT.read()
    soup = BeautifulSoup(source_code, "html.parser")

    for t in soup.findAll('topic'):
        q = t.find('query').text
        Queries[int(t['number'])] = q
    QT.close()
    processQuery()


# In[6]:


def getTermIDs():
    termID = open("F:\\PycharmProjects\\SearchEngine\\files\\termids.txt", mode="r", encoding="utf-8", errors='ignore')
    for line in termID:
        sen = line.split()
        termIDs[int(sen[0])] = sen[1]
    termID.close()


# In[7]:


def getDocIDs():
    docID = open("F:\\PycharmProjects\\SearchEngine\\files\\docids.txt", mode="r", encoding="utf-8", errors='ignore')

    for line in docID:
        sen = line.split()
        docIDs[int(sen[0])] = sen[1]
    docID.close()


# In[8]:


# total terms in a doc
def getDocInfo():
    doc_ind = open("F:\\PycharmProjects\\SearchEngine\\files\\doc_index.txt", mode="r", encoding="utf-8",
                   errors='ignore')

    tCorpusTerms = 0
    did = 1
    for line in doc_ind:
        sen = line.split()
        tCorpusTerms = tCorpusTerms + len(sen) - 2
        if int(sen[0]) == did:
            #         if did == 1:
            #             print(sen[0])
            if did in tDocTerms:
                tDocTerms[did] = tDocTerms[did] + len(sen) - 2
            else:
                tDocTerms[did] = len(sen) - 2
        else:
            did = did + 1
            tDocTerms[did] = len(sen) - 2
    doc_ind.close()
    return tCorpusTerms


# In[9]:


def avgQueryLength():
    avg_qlen = 0
    for val in Queries:
        lst = Queries[val]
        avg_qlen = avg_qlen + len(lst)
    avg_qlen = avg_qlen / len(Queries)
    return avg_qlen


# In[10]:


# In[11]:


def terminfo(id):
    term_info = open("F:\\PycharmProjects\\SearchEngine\\files\\term_info.txt", 'r', encoding='utf8', errors='ignore')
    for line in term_info:
        sentence = line.split()
        if int(sentence[0]) == id:
            break
    term_info.close()
    return sentence


# In[12]:


# term Document info
def getTermInfo(offset, ind):
    term_ind = open("F:\\PycharmProjects\\SearchEngine\\files\\term_index.txt", mode="r", encoding="utf-8",
                    errors='ignore')
    term_ind.seek(offset - 1)
    sentence = term_ind.readline()
    words = sentence.split()
    docid = []
    for i in range(1, len(words)):
        docid.append(words[i].split(":"))
    st = 0
    count = 0
    positions = []

    cont = True
    for i in range(0, len(docid)):
        if int(docid[i][0]) != 0:
            st = st + int(docid[i][0])
        if st == ind:
            decode = int(docid[i][1])
            while cont:
                positions.append(docid[i][1])
                count += 1
                i += 1
                if i != len(docid):
                    docid[i][1] = str(int(docid[i][1]) + decode)
                    if int(docid[i][0]) != 0:
                        cont = False
                else:
                    cont = False
            break
    positions.append(count)
    term_ind.close()
    return positions


# In[13]:


def OkapiTF():
    oktfFile = open("F:\\PycharmProjects\\SearchEngine\\files\\okapiTFRanked.txt", mode="w", encoding="utf-8",
                    errors='ignore')
    for key, val in Queries.items():
        # rank = 0
        temp_dict = {}
        for i in range(1, len(docIDs) + 1):
            # extract key
            # rank = rank + 1
            cross_prod = []
            doc_len = []
            query_len = []
            for j in range(0, len(val)):  # for all words in a query
                for k, v in termIDs.items():
                    if v == val[j]:
                        temp = terminfo(k)  # for offset
                        t_offset = temp[1]
                        td_info = getTermInfo(int(t_offset), int(i))
                        length = len(td_info)
                        tf = td_info[length - 1]
                        oktf_doc = tf / (tf + 0.5 + (1.5 * tDocTerms[i] / avgDocLen))
                        t = oktf_doc * oktf_doc
                        doc_len.append(t)

                        tf_q = val.count(v)
                        oktf_q = tf_q / (tf_q + 0.5 + (1.5 * len(val) / avg_qlen))
                        t = oktf_q * oktf_q
                        query_len.append(t)

                        oktf = oktf_doc * oktf_q
                        cross_prod.append(oktf)
            numerator = sum(cross_prod)
            d_len = math.sqrt(sum(doc_len))
            q_len = math.sqrt(sum(query_len))
            if d_len > 0 and numerator > 0:
                score = numerator / (d_len * q_len)
            else:
                score = 0
            temp_dict[docIDs[i]] = score
        # converted to list when sorted
        temp_dict = sorted(temp_dict.items(), key=operator.itemgetter(1), reverse=True)

        for i in range(len(temp_dict)):
            oktfFile.write(str(key))
            oktfFile.write("\t")
            oktfFile.write("0")
            oktfFile.write("\t")
            oktfFile.write(temp_dict[i][0])
            oktfFile.write("\t")
            oktfFile.write(str(i + 1))
            oktfFile.write("\t")
            oktfFile.write(str(temp_dict[i][1]))
            oktfFile.write("\t")
            oktfFile.write("run1")
            oktfFile.write("\n")
    oktfFile.close()


# In[14]:


# In[13]:


def TF_IDF():
    TFIDF_file = open("F:\\PycharmProjects\\SearchEngine\\files\\TF_IDFranked.txt", mode="w", encoding="utf-8",
                      errors='ignore')
    for key, val in Queries.items():
        # rank = 0
        temp_dict = {}
        for i in range(1, len(docIDs) + 1):
            # extract key
            # rank = rank + 1
            cross_prod = []
            doc_len = []
            query_len = []
            for j in range(0, len(val)):  # for all words in a query
                for k, v in termIDs.items():
                    if v == val[j]:
                        temp = terminfo(k)  # for offset
                        t_offset = temp[1]
                        td_info = getTermInfo(int(t_offset), int(i))
                        length = len(td_info)
                        tf = td_info[length - 1]
                        oktf_doc = tf / (tf + 0.5 + (1.5 * tDocTerms[i] / avgDocLen))
                        t = oktf_doc * oktf_doc
                        doc_len.append(t)

                        tf_q = val.count(v)
                        oktf_q = tf_q / (tf_q + 0.5 + (1.5 * len(val) / avg_qlen))
                        t = oktf_q * oktf_q
                        query_len.append(t)

                        t_info = terminfo(k)
                        oktf = oktf_doc * oktf_q * math.log10((len(docIDs) / int(temp[3])))  # t_info[2]
                        cross_prod.append(oktf)
            numerator = sum(cross_prod)
            d_len = math.sqrt(sum(doc_len))
            q_len = math.sqrt(sum(query_len))
            if d_len > 0 and numerator > 0:
                score = numerator / (d_len * q_len)
            else:
                score = 0
            temp_dict[docIDs[i]] = score
        # converted to list when sorted
        temp_dict = sorted(temp_dict.items(), key=operator.itemgetter(1), reverse=True)

        for i in range(len(temp_dict)):
            TFIDF_file.write(str(key))
            TFIDF_file.write("\t")
            TFIDF_file.write("0")
            TFIDF_file.write("\t")
            TFIDF_file.write(temp_dict[i][0])
            TFIDF_file.write("\t")
            TFIDF_file.write(str(i + 1))
            TFIDF_file.write("\t")
            TFIDF_file.write(str(temp_dict[i][1]))
            TFIDF_file.write("\t")
            TFIDF_file.write("run1")
            TFIDF_file.write("\n")
    TFIDF_file.close()


# In[14]:


# In[17]:


def BM25():
    BM25_file = open("F:\\PycharmProjects\\SearchEngine\\files\\BM25_ranked.txt", mode="w", encoding="utf-8",
                     errors='ignore')

    k1 = 1.2
    k2 = 120
    b = 0.75

    for key, val in Queries.items():
        temp_dict = {}
        for i in range(1, len(docIDs) + 1):  # for testing i have taken only the first 10 docs
            score = 0
            K_ = k1 * ((1 - b) + (b * tDocTerms[i] / avgDocLen))
            for j in range(0, len(val)):  # for all words in a query
                for k, v in termIDs.items():
                    if v == val[j]:
                        temp = terminfo(k)  # for offset
                        t_offset = temp[1]
                        td_info = getTermInfo(int(t_offset), int(i))
                        length = len(td_info)
                        tf = td_info[length - 1]
                        tf_q = val.count(v)

                        first = (len(docIDs) + 0.5) / (int(temp[3]) + 0.5)
                        second = ((1 + k1) * tf) / (K_ + tf)
                        third = ((1 + k1) * tf_q) / (k2 + tf_q)
                        score = score + (math.log10(first) * second * third)

            temp_dict[docIDs[i]] = score

        # converted to list when sorted
        temp_dict = sorted(temp_dict.items(), key=operator.itemgetter(1), reverse=True)

        for i in range(len(temp_dict)):
            BM25_file.write(str(key))
            BM25_file.write("\t")
            BM25_file.write("0")
            BM25_file.write("\t")
            BM25_file.write(temp_dict[i][0])
            BM25_file.write("\t")
            BM25_file.write(str(i + 1))
            BM25_file.write("\t")
            BM25_file.write(str(temp_dict[i][1]))
            BM25_file.write("\t")
            BM25_file.write("run1")
            BM25_file.write("\n")
    BM25_file.close()


# In[18]:


# In[19]:


def Jelinek_Mercer():
    JM_file = open("F:\\PycharmProjects\\SearchEngine\\files\\JM_ranked.txt", mode="w", encoding="utf-8",
                   errors='ignore')
    lambdaa = 0.6
    for key, val in Queries.items():
        temp_dict = {}
        for i in range(1, len(docIDs) + 1):
            score = 1
            for j in range(0, len(val)):  # for all words in a query
                for k, v in termIDs.items():
                    if v == val[j]:
                        temp = terminfo(k)  # for offset
                        t_offset = temp[1]
                        td_info = getTermInfo(int(t_offset), int(i))
                        length = len(td_info)
                        tf = td_info[length - 1]

                        first = lambdaa * tf / tDocTerms[i]
                        second = (1 - lambdaa) * int(temp[2]) / tCorpusTerms
                        score = score * (first + second)

            temp_dict[docIDs[i]] = score
        # converted to list when sorted
        temp_dict = sorted(temp_dict.items(), key=operator.itemgetter(1), reverse=True)

        for i in range(len(temp_dict)):
            JM_file.write(str(key))
            JM_file.write("\t")
            JM_file.write("0")
            JM_file.write("\t")
            JM_file.write(temp_dict[i][0])
            JM_file.write("\t")
            JM_file.write(str(i + 1))
            JM_file.write("\t")
            JM_file.write(str(temp_dict[i][1]))
            JM_file.write("\t")
            JM_file.write("run1")
            JM_file.write("\n")
    JM_file.close()


# In[20]:


print("Running")
parseQueryTopics()
getTermIDs()
getDocIDs()
tCorpusTerms = getDocInfo()
avgDocLen = tCorpusTerms / len(docIDs)
avg_qlen = avgQueryLength()

command = input("Enter command.\n")
loop = True
while loop:
    if command == 'exit':
        loop = False

    elif command == '--score TF':
        OkapiTF()

    elif command == '--score TF-IDF':
        TF_IDF()

    elif command == '--score BM25':
        BM25()

    elif command == '--score JM':
        Jelinek_Mercer()

    command = input("Enter command.\n")