diff --git a/.env.example b/.env.example
index b9a00a7..c73f646 100644
--- a/.env.example
+++ b/.env.example
@@ -6,4 +6,6 @@ SPOKEN_DB='13thJune'
 SECRET_KEY=''
 VIDEO_PATH=''
 DEBUG=True
-TEMPLATE_DEBUG=True
\ No newline at end of file
+TEMPLATE_DEBUG=True
+URL_MONGO_IP='localhost'
+URL_MONGO_PORT='27017'
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 9ebc923..6aa15a5 100755
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ static/CACHE/*
 robots.txt
 # C extensions
 *.so
+media/
 
 # Packages
 *.egg
@@ -43,4 +44,4 @@ nosetests.xml
 .pydevproject
 uwsgi-master.pid
 .env
-.vscode
\ No newline at end of file
+.vscode
diff --git a/requirements.txt b/requirements.txt
index 08937ec..bcd2b99 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,5 +13,12 @@ django-extensions==2.1.3
 django-filter==2.0.0
 django-debug-toolbar==1.4
 python-dotenv==0.10.3
+scikit-learn>=0.19.1
+StackAPI==0.1.12
+pymongo==3.10.1
+lxml==4.6.2
+pandas==0.25.3
 nltk==3.5
+scipy==1.4.1
 sklearn==0.0
+django-mysql==3.10.0
diff --git a/spoken_auth/models.py b/spoken_auth/models.py
index f7bed1b..3fc4a7b 100755
--- a/spoken_auth/models.py
+++ b/spoken_auth/models.py
@@ -82,3 +82,12 @@ class TutorialResources(models.Model):
 
     class Meta:
         db_table = 'creation_tutorialresource'
+
+
+class TutorialCommonContent(models.Model):
+    id = models.IntegerField(primary_key=True)
+    tutorial_detail = models.ForeignKey(TutorialDetails, on_delete=models.CASCADE,)
+    keyword = models.TextField()
+
+    class Meta:
+        db_table = 'creation_tutorialcommoncontent'
diff --git a/static/website/js/custom.js b/static/website/js/custom.js
index d99416f..8b96d7b 100755
--- a/static/website/js/custom.js
+++ b/static/website/js/custom.js
@@ -66,7 +66,7 @@ $(document).ready(function() {
         }
     });
 
-    $tutorial.change(function() {
+    $tutorial.on('input', function() {
         /* resetting dropdowns */
         reset("minute_range", "second_range");
         var category = $category.val();
@@ -98,6 +98,35 @@ $(document).ready(function() {
             });
         }
     });
+
+    // this is in faq page
+    $tutorial.change(function() {
+        $.ajax({
+            url: "/ajax-faq-questions/",
+            type: "POST",
+            data: {
+                category: $category.val(),
+                tutorial: $tutorial.val(),
+            },
+            dataType: "html",
+            success: function(data) {
+                $response = $(data);
+                var similar_count= $response.find("#similar-count").text();
+                console.log(similar_count);
+                $("#similar-link").show();
+                $("#modal-body").html(data);
+                
+                $.ajax({
+                    url:"/ajax-fetch-questions/",
+                    type: "POST",
+                    data: {
+                        category: $category.val(),
+                        tutorial: $tutorial.val(),
+                    }
+                });
+            }
+        });
+    });    
     
     $title.keyup(function() {
         var len = $title.val().split(' ').length;
@@ -117,7 +146,8 @@ $(document).ready(function() {
             success: function(data) {
                 $response = $(data);
                 var similar_count= $response.find("#similar-count").text();
-                $("#similar-link").show().html(similar_count);
+                console.log(similar_count);
+                $("#similar-link").show();
                 $("#modal-body").html(data);
             }
         });
diff --git a/static/website/templates/ajax-similar-questions.html b/static/website/templates/ajax-similar-questions.html
index c3890ab..ad81963 100755
--- a/static/website/templates/ajax-similar-questions.html
+++ b/static/website/templates/ajax-similar-questions.html
@@ -9,13 +9,17 @@
     {% endif %}
 </div>
 <br>
+
 <table class="table table-bordered table-hover">
 {% for question in questions %}
     <tr>
         <td>
-            <a href="{% url 'website:get_question' question.id %}" target="_blank">
+            <a href="{{ question.body }}" target="_blank">
                 {{ question.title }}
             </a>
+            {% for tag in question.tags %}
+            <span class="badge badge-pill badge-dark">{{ tag }}</span>
+            {% endfor %}
         </td>
     </tr>
 {% endfor %}
diff --git a/static/website/templates/ajax_faq_questions.html b/static/website/templates/ajax_faq_questions.html
new file mode 100644
index 0000000..07e4adf
--- /dev/null
+++ b/static/website/templates/ajax_faq_questions.html
@@ -0,0 +1,57 @@
+<div>
+    <div id="similar-count">
+        {% if questions.count > 0 %}
+            {% if questions.count == 1 %}
+                1 similar question
+            {% else %}
+                {{ questions.count }} similar questions
+            {% endif %}
+        {% endif %}
+    </div>
+    <br>
+    <table class="table table-bordered table-hover">
+    {% if questions|length == 0 %}
+        <h4>No similar questions found</h4>
+    {% else %}
+        <thead>
+            <tr>
+                <th>Accepted <br> Answer</th>
+                <th>Author <br> Details</th>
+                <th style="vertical-align: middle;">Question Details</th>
+            </tr>
+        </thead>
+    {% endif %}
+    
+    {% for question in questions %}
+        <tr>
+            <td style="text-align: center;">
+                {% if question.accepted_answer != "" %}
+                    <a href="{{ question.accepted_answer }}" target="_blank"><i style="color: green" class="fa fa-lg fa-check" aria-hidden="true"></i></a>
+                {% endif %}
+            </td>
+            <td style="text-align: center;">
+                <a href="{{ question.owner.link }}" target="_blank"><i class="fa fa-md fa-user-check" aria-hidden="true"></i></a>
+            </td>
+            <td>
+                <div class="col-md-8">
+                    <a href="{{ question.body }}" target="_blank">
+                        &nbsp; {{ question.title }}
+                    </a>
+                    <br>
+                    {% for tag in question.tags %}
+                    <span class="badge badge-pill badge-dark">{{ tag }}</span>
+                    {% endfor %}
+                </div>
+                <div class="col-md-4">
+                    <div class="row pull-right" style="font-size: 14;">
+                        <span class="label label-warning">{{ question.views }} <i class="fa fa-md fa-eye" aria-hidden="true"></i></span>
+                        <span class="label label-success">{{ question.score }} <i class="fa fa-md fa-thumbs-up" aria-hidden="true"></i></span>
+                        <span class="label label-primary">{{ question.answer_count }} <i class="fa fa-md fa-comments" aria-hidden="true"></i></span>
+                    </div>
+                </div>
+            </td>
+        </tr>
+    {% endfor %}
+    </table>
+    </div> <!-- /div for ajax -->
+    
\ No newline at end of file
diff --git a/static/website/templates/base.html b/static/website/templates/base.html
index 0152830..32c7671 100644
--- a/static/website/templates/base.html
+++ b/static/website/templates/base.html
@@ -17,7 +17,8 @@
          
         {% endcompress %}
 				<script src="{% static 'website/js/jquery-1.12.4.min.js' %}"></script>
-				<script src="{% static 'website/js/jquery.tablesorter.min.js' %}"></script>
+                <script src="{% static 'website/js/jquery.tablesorter.min.js' %}"></script>
+                <script src="https://kit.fontawesome.com/9e55978135.js" crossorigin="anonymous"></script>
         <!-- settig up csrf token for every ajax request -->
         <script type="text/javascript">window.csrftoken = '{{ csrf_token }}';
           function csrfSafeMethod(method) {
@@ -70,6 +71,12 @@
                                     Search
                                 </a>
                               </li>
+                              <li>
+                                <a href="{% url 'website:faq' %}">
+                                    <span class="glyphicon glyphicon-list"></span>
+                                    FAQ
+                                </a>
+                              </li>
                               <li>
                                 <a href="{% url 'website:new_question' %}">
                                     <span class="glyphicon glyphicon-pencil"></span> 
diff --git a/static/website/templates/faq.html b/static/website/templates/faq.html
new file mode 100644
index 0000000..2829e0e
--- /dev/null
+++ b/static/website/templates/faq.html
@@ -0,0 +1,65 @@
+{% extends 'website/templates/base.html' %}
+{% load static %}
+{% load widget_tweaks %}
+{% block content %}
+    <h4>
+        <span class="glyphicon glyphicon-pencil">
+        </span> Create a new question . . .
+    </h4>
+    <hr>
+    <form role="form" action="" method="POST" novalidate>{% csrf_token %}
+    {% with WIDGET_ERROR_CLASS='field_error' %}
+
+    <p>
+    Please enter the tutorial details.
+        <img id="ajax-loader" src="{% static 'website/images/ajax-loader.gif' %}" style="display:none;">
+    </p>
+    <div class="row">
+        <div class="col-lg-3 col-md-3 col-sm-3">
+            {% render_field form.category class+="form-control"%} 
+        </div>
+        <div class="col-lg-3 col-md-3 col-sm-3">
+            {% if category %}
+                {% render_field form.tutorial class+="form-control" %} 
+            {% else %}
+                {% render_field form.tutorial class+="form-control" disabled="disabled" %} 
+            {% endif %}
+        </div>
+
+        <!-- <div class="col-lg-2 col-md-2 col-sm-2">
+            <small><strong>
+            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
+                0 similar questions
+            </a>
+            </strong></small>
+        </div> -->
+    </div>
+    <hr>
+    <!-- Added this part for fetching related questions -->
+    <div class="alert-dismissible" id="similar-link">
+        <div class="modal-header">
+            <h4 class="modal-title" id="myModalLabel"><i style="color: #f48024;" class="fa fa-lg fa-stack-overflow" aria-hidden="true"></i> Related Questions from StackOverflow</h4>
+        </div>
+        <div class="modal-body" id="modal-body" style="color: black">
+            ...
+        </div>
+    </div>
+
+    
+    {% endwith %}
+    </form>
+
+<script src="{% static 'website/js/nicEdit.js' %}" type="text/javascript"></script>
+<script type="text/javascript">
+bkLib.onDomLoaded(function() {
+	new nicEditor({
+        iconsPath: "{% static 'website/js/nicEditorIcons.gif' %}",
+        buttonList : ['fontSize','bold','italic','underline','strikeThrough','subscript','superscript','html','image', 'link', 'forecolor', 'bgcolor']
+    }).panelInstance('id_body');
+});
+</script>
+{% endblock %}
+
+{% block javascript %}
+    <script src="{% static 'website/js/custom.js' %}"></script>
+{% endblock %}
diff --git a/static/website/templates/new-question.html b/static/website/templates/new-question.html
index c088447..4f36944 100755
--- a/static/website/templates/new-question.html
+++ b/static/website/templates/new-question.html
@@ -31,6 +31,13 @@ <h4>
         <div class="col-lg-2 col-md-2 col-sm-2">
             {% render_field form.second_range class+="form-control" disabled="disabled" %} 
         </div>
+        <!-- <div class="col-lg-2 col-md-2 col-sm-2">
+            <small><strong>
+            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
+                0 similar questions
+            </a>
+            </strong></small>
+        </div> -->
     </div>
     <hr>
 
@@ -61,7 +68,7 @@ <h4>
     </form>
 
     <!-- Modal -->
-    <div class="modal fade" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
+    <!-- <div class="modal fade" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
       <div class="modal-dialog">
         <div class="modal-content">
           <div class="modal-header">
@@ -70,10 +77,10 @@ <h4 class="modal-title" id="myModalLabel">Similar Questions</h4>
           </div>
           <div class="modal-body" id="modal-body">
             ...
-          </div>
-        </div><!-- /.modal-content -->
-      </div><!-- /.modal-dialog -->
-    </div><!-- /.modal -->
+          </div> -->
+        <!-- </div>/.modal-content -->
+      <!-- </div>/.modal-dialog -->
+    <!-- </div>/.modal -->
 
 <script src="{% static 'website/js/nicEdit.js' %}" type="text/javascript"></script>
 <script type="text/javascript">
diff --git a/website/forms.py b/website/forms.py
index c7d5232..d9d8d1b 100755
--- a/website/forms.py
+++ b/website/forms.py
@@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs):
             seconds = (
                 ("", "sec"),
             )
-
+        
         if not category and args and 'category' in args[0] and args[0]['category']:
             category = args[0]['category']
         if FossCategory.objects.filter(foss=category).exists():
@@ -66,7 +66,6 @@ def __init__(self, *args, **kwargs):
             self.fields['minute_range'] = forms.CharField(widget=forms.Select(choices=minutes))
             self.fields['second_range'] = forms.CharField(widget=forms.Select(choices=seconds))
 
-
 class AnswerQuesitionForm(forms.Form):
     question = forms.IntegerField(widget=forms.HiddenInput())
     body = forms.CharField(widget=forms.Textarea())
diff --git a/website/helpers.py b/website/helpers.py
index 5db0a53..bddcaad 100755
--- a/website/helpers.py
+++ b/website/helpers.py
@@ -45,8 +45,9 @@ def prettify(string):
 
 def pre_process(text):
     text=text.lower()                       # lowercase
-    text=re.sub("<!--?.*?-->","",text)      # remove tags
-    text=re.sub("(\\d|\\W)+"," ",text)      # remove special characters and digits
+    text = re.sub('<.*?>', ' ', text).replace('  ',' ')
+    text=re.sub("<!--?.*?-->"," ",text).replace('  ',' ')      # remove tags
+    text=re.sub("(\\d|\\W)+"," ",text).replace('  ',' ')      # remove special characters and digits
     return text
 
 def clean_user_data(text):
diff --git a/website/migrations/0001_initial.py b/website/migrations/0001_initial.py
new file mode 100644
index 0000000..d65fb65
--- /dev/null
+++ b/website/migrations/0001_initial.py
@@ -0,0 +1,28 @@
+# Generated by Django 2.2.6 on 2021-01-07 04:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_mysql.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StackQuestions',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('category', models.CharField(max_length=200)),
+                ('tutorial', models.CharField(max_length=200)),
+                ('stackdata', django_mysql.models.JSONField(default=dict)),
+            ],
+            options={
+                'unique_together': {('category', 'tutorial')},
+            },
+        )
+    ]
diff --git a/website/migrations/__init__.py b/website/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/website/models.py b/website/models.py
index 9dbf6f0..8f5586a 100755
--- a/website/models.py
+++ b/website/models.py
@@ -1,5 +1,6 @@
 from django.db import models
 from django.contrib.auth import get_user_model
+from django_mysql.models import JSONField
 
 User = get_user_model()
 
@@ -87,4 +88,11 @@ def poster(self):
         user = User.objects.get(id=self.pid)
         return user.username
 
+class StackQuestions(models.Model):
+    category = models.CharField(max_length=200)
+    tutorial = models.CharField(max_length=200)
+    stackdata = JSONField()
+
+    class Meta(object):
+        unique_together = ('category', 'tutorial')
 # CDEEP database created using inspectdb arg of manage.py
diff --git a/website/populate_mongo.py b/website/populate_mongo.py
new file mode 100644
index 0000000..7693312
--- /dev/null
+++ b/website/populate_mongo.py
@@ -0,0 +1,102 @@
+import json
+import pandas
+import os
+import pymongo
+from stackapi import StackAPI
+    
+from django.template.context_processors import csrf
+from django.db.models import Q, Count
+from django.contrib.auth import get_user_model
+from django.conf  import settings
+
+from spoken_auth.models import TutorialDetails, TutorialResources, TutorialCommonContent
+from .tfidf import *
+
+
+def get_questions_from_stack(category, tutorial, terms, db_tags):
+    rel_tags = []
+
+    if len(db_tags) != 0:
+        rel_tags.extend(db_tags)
+    if len(terms) != 0:
+        rel_tags.extend(terms)
+
+    print("rel_tags")
+    print(rel_tags)
+    mongo_ip = os.getenv("URL_MONGO_IP")
+    mongo_port = os.getenv("URL_MONGO_PORT")
+    mongo_url = "mongodb://" + mongo_ip + ":" + mongo_port
+    client = pymongo.MongoClient(mongo_url)
+    db = client.stackapi
+    collec_ques = db.questions
+    collec_ques.create_index('question_id')
+
+    ''' fetch questions '''
+    entries = []
+    print("Fetching data from stackoverflow.. ")
+    SITE = StackAPI('stackoverflow')
+    
+    ''' Fetching questions with only category name(e.g. latex) as tag '''
+    questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category, sort='votes', order='desc')
+    entries.extend(questions['items'])
+    
+    ''' Fetching questions with addition of rel_tags(obtained from quert/srt files) '''
+    for t in db_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+        
+    for t in rel_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+    
+    ''' inserting fetched data into mongodb '''
+    if len(entries) > 0:
+        for entry in entries:
+            collec_ques.update({'question_id': entry['question_id']}, entry, upsert=True)
+    print(str(len(entries)) + " questions fetched and inserted into mongodb")
+
+def get_categories_tutorials():
+    ''' get all categories '''
+    categories = []
+    trs = TutorialResources.objects.filter(Q(status=1) | Q(status=2),tutorial_detail__foss__show_on_homepage__lt=2, language__name='English')
+    trs = trs.values('tutorial_detail__foss__foss').order_by('tutorial_detail__foss__foss')
+        
+    for tr in trs.values_list('tutorial_detail__foss__foss').distinct():
+        categories.append(tr[0])
+    
+    print("categories:")
+    print(categories)
+    ''' for each category fetch all tutorials '''
+    for category in categories:
+        tutorials = TutorialDetails.objects.using('spoken').filter(foss__foss=category).order_by('level', 'order')
+        cat = tutorials[0].foss
+        path = settings.MEDIA_ROOT + 'videos/' + str(cat.pk)
+        create_vocab_tfidf(path)
+        print("for categoery - " + str(cat))
+
+        ''' for each tutorial in the category, find terms from db as well as srt files '''
+        for tut in tutorials:
+            tutorial = tut.tutorial
+            print("tutorial - " + str(tutorial))
+            td_rec = TutorialDetails.objects.using('spoken').filter(tutorial=tutorial, foss__foss=category).order_by('level', 'order')
+            cat = td_rec[0].foss
+            td = td_rec[0]
+
+            ''' fetching terms from db '''
+            db_tags = TutorialCommonContent.objects.using('spoken').filter(tutorial_detail=td.pk)
+            db_tags = db_tags[0].keyword.replace(".", "").split(", ")
+            print("list of db_tags >>>>>>>>>>>>")
+            print(db_tags)
+            
+            ''' fetching terms from srt files using NLP method '''
+            filename = settings.MEDIA_ROOT + 'videos/' + str(cat.pk) + '/' + str(td.pk) + '/' + tutorial.replace(' ', '-') + '-English.srt'
+            topic_keys = extract_keywords(filename)
+            print("topic_keys >>>>>>>>>>>>>")
+            print(topic_keys)
+
+            ''' Scraping stackoverflow to get questions and store in mongodb '''
+            get_questions_from_stack(category.lower(), tutorial.lower(), topic_keys, db_tags)
+            
+    print("Successfully fetched data from stackoverflow")
+
+get_categories_tutorials()
\ No newline at end of file
diff --git a/website/tfidf.py b/website/tfidf.py
new file mode 100644
index 0000000..9a23e3c
--- /dev/null
+++ b/website/tfidf.py
@@ -0,0 +1,99 @@
+import sys, os, re
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from nltk.corpus import stopwords
+
+def pre_process(text):    
+    text=text.lower()                       # lowercase
+    text=re.sub("<!--?.*?-->","",text)      # remove tags
+    text=re.sub("(\\d|\\W)+"," ",text)      # remove special characters and digits
+    return text
+ 
+ 
+# load a set of stop words
+stopwords = stopwords.words('english')
+
+cv = ""
+tfidf_transformer = ""
+feature_names = ""
+
+def create_vocab_tfidf(dirpath):
+    global cv
+    global tfidf_transformer
+    global feature_names
+    filenames = []
+    docs = []
+    listOfDir = os.listdir(dirpath)
+    for dir in listOfDir:
+        path = os.path.join(dirpath, dir)
+        for file in os.listdir(path):
+            filenames.append(file.split(".")[0].replace('-', " "))
+            with open(path+"/"+file, "r") as f:
+                doc = ' '.join([str(line) for line in f.readlines()]) 
+                docs.append(doc)
+    
+    df_idf = pd.DataFrame()
+    df_idf['title'] = filenames
+    print(df_idf['title'])
+    df_idf['body'] = docs
+    df_idf['text'] = df_idf['title'] + df_idf['body']
+    df_idf['text'] = df_idf['text'].apply(lambda x:pre_process(x))
+
+    # get the text column 
+    docs=df_idf['text'].tolist()
+
+    # create a vocabulary of words, ignore words that appear in 85% of documents, eliminate stop words
+    cv=CountVectorizer(max_df=0.85,stop_words=stopwords)
+    word_count_vector=cv.fit_transform(docs)
+    print(list(cv.vocabulary_.keys())[:50])
+
+    tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)
+    tfidf_transformer.fit(word_count_vector)
+
+    # this is done only once, this is a mapping of index to words 
+    feature_names=cv.get_feature_names()
+
+
+####################################################################################################
+
+
+def sort_coo(coo_matrix):
+    tuples = zip(coo_matrix.col, coo_matrix.data)
+    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)
+ 
+def extract_topn_from_vector(feature_names, sorted_items, topn):
+    """get the feature names and tf-idf score of top n items"""
+    
+    sorted_items = sorted_items[:topn]       # use only topn items from vector
+    score_vals = []
+    feature_vals = []
+    
+    # word index and corresponding tf-idf score
+    for idx, score in sorted_items:
+        #keep track of feature name and its corresponding score
+        score_vals.append(round(score, 3))
+        feature_vals.append(feature_names[idx])
+ 
+    #create a tuples of feature,score
+    results= {}
+    for idx in range(len(feature_vals)):
+        results[feature_vals[idx]]=score_vals[idx]
+    return results
+
+
+# extract keywords
+def extract_keywords(filename):
+    print(filename)
+    with open(filename, "r") as f:
+        doc = ' '.join([str(line) for line in f.readlines()])
+    
+    #generate tf-idf for the given document
+    tf_idf_vector=tfidf_transformer.transform(cv.transform([doc]))
+    
+    #sort the tf-idf vectors by descending order of scores
+    sorted_items=sort_coo(tf_idf_vector.tocoo())
+    
+    #extract only the top n; n is given as the last argument
+    keywords=extract_topn_from_vector(feature_names,sorted_items, 5)
+    print(keywords)
+    return list(keywords.keys())
\ No newline at end of file
diff --git a/website/urls.py b/website/urls.py
index 08decec..a83e8c8 100755
--- a/website/urls.py
+++ b/website/urls.py
@@ -21,6 +21,7 @@
     url(r'^user/(?P<user_id>\d+)/answers/$', views.user_answers, name='user_answers'),
     url(r'^clear-notifications/$', views.clear_notifications, name='clear_notifications'),
     url(r'^search/$', views.search, name='search'),
+    url(r'^faq/$', views.faq, name='faq'),
     url(r'^unanswered-notification/$', views.unanswered_notification, name='unanswered_notification'),
 
     # Ajax helpers
@@ -33,6 +34,8 @@
     url(r'^ajax-answer-comment-update/$', views.ajax_answer_comment_update,
         name='ajax_answer_comment_update'),
     url(r'^ajax-similar-questions/$', views.ajax_similar_questions, name='ajax_similar_questions'),
+    url(r'^ajax-faq-questions/$', views.ajax_faq_questions, name='ajax_faq_questions'),
+    url(r'^ajax-fetch-questions/$', views.ajax_fetch_questions, name='ajax_fetch_questions'),
     url(r'^ajax-notification-remove/$', views.ajax_notification_remove, name='ajax_notification_remove'),
     url(r'^ajax-keyword-search/$', views.ajax_keyword_search, name='ajax_keyword_search'),
     url(r'^ajax-time-search/$', views.ajax_time_search, name='ajax_time_search'),
diff --git a/website/views.py b/website/views.py
index 49213d4..fb614f1 100755
--- a/website/views.py
+++ b/website/views.py
@@ -1,5 +1,9 @@
 import json
-
+#import pandas
+import os
+import pymongo
+from stackapi import StackAPI
+    
 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden
 from django.shortcuts import render, get_object_or_404
 from django.template.context_processors import csrf
@@ -9,8 +13,8 @@
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.contrib.auth import get_user_model
 
-from website.models import Question, Answer, Notification, AnswerComment
-from spoken_auth.models import TutorialDetails, TutorialResources
+from website.models import Question, Answer, Notification, AnswerComment, StackQuestions
+from spoken_auth.models import TutorialDetails, TutorialResources, TutorialCommonContent
 from website.forms import NewQuestionForm, AnswerQuesitionForm
 from website.helpers import get_video_info, prettify, clean_user_data, get_similar_questions
 from django.conf  import settings
@@ -19,12 +23,16 @@
 from .sortable import SortableHeader, get_sorted_list, get_field_index
 from django.db.models import Count
 
+from .tfidf import *
+
 
 User = get_user_model()
 categories = []
 trs = TutorialResources.objects.filter(Q(status=1) | Q(status=2),tutorial_detail__foss__show_on_homepage__lt=2, language__name='English')
-trs = trs.values('tutorial_detail__foss__foss').order_by('tutorial_detail__foss__foss')
+cat_tutorials = []
 
+trs = trs.values('tutorial_detail__foss__foss').order_by('tutorial_detail__foss__foss')
+    
 for tr in trs.values_list('tutorial_detail__foss__foss').distinct():
     categories.append(tr[0])
 
@@ -411,22 +419,42 @@ def search(request):
     }
     return render(request, 'website/templates/search.html', context)
 
+def faq(request):
+    context = {}
+    category = request.GET.get('category', None)
+    tutorial = request.GET.get('tutorial', None)
+    minute_range = request.GET.get('minute_range', None)
+    second_range = request.GET.get('second_range', None)
+    # pass minute_range and second_range value to NewQuestionForm to populate on select
+    form = NewQuestionForm(category=category, tutorial=tutorial,
+                            minute_range=minute_range, second_range=second_range)
+    context['category'] = category
+
+    context['form'] = form
+    context.update(csrf(request))
+    return render(request, 'website/templates/faq.html', context)
+
+
 # Ajax Section
 # All the ajax views go below
 
-
 def ajax_category(request):
     context = {
         'categories': categories
     }
     return render(request, 'website/templates/ajax_categories.html', context)
 
-
 def ajax_tutorials(request):
     if request.method == 'POST':
         category = request.POST.get('category')
         tutorials = TutorialDetails.objects.using('spoken').filter(
             foss__foss=category).order_by('level', 'order')
+        cat = tutorials[0].foss
+        #path = settings.VIDEO_PATH + str(cat.pk)
+        path = settings.VIDEO_PATH+str(cat.pk)+'/'
+        create_vocab_tfidf(path)
+        for tut in tutorials:
+            cat_tutorials.append(tut.tutorial)
         context = {
             'tutorials': tutorials
         }
@@ -452,19 +480,19 @@ def ajax_duration(request):
             video_resource.video
         )
 
-        video_info = get_video_info(video_path)
+        # video_info = get_video_info(video_path)
 
-        # convert minutes to 1 if less than 0
-        # convert seconds to nearest upper 10th number eg(23->30)
-        minutes = video_info['minutes']
-        seconds = video_info['seconds']
-        if minutes < 0:
-            minutes = 1
-        seconds = int(seconds - (seconds % 10 - 10))
-        seconds = 60
+        # # convert minutes to 1 if less than 0
+        # # convert seconds to nearest upper 10th number eg(23->30)
+        # minutes = video_info['minutes']
+        # seconds = video_info['seconds']
+        # if minutes < 0:
+        #     minutes = 1
+        # seconds = int(seconds - (seconds % 10 - 10))
+        # seconds = 60
         context = {
-            'minutes': minutes,
-            'seconds': seconds,
+            'minutes': 5,
+            'seconds': 6,
         }
         return render(request, 'website/templates/ajax-duration.html', context)
 
@@ -546,6 +574,195 @@ def ajax_answer_comment_update(request):
 
     return HttpResponseForbidden("Not Authorised")
 
+def get_questions_from_stack(category, tutorial, query, terms, db_tags):
+    rel_tags = []
+
+    # this part is run when the function is called from FAQ page with category and tutorial selected, here query=""
+    if len(db_tags) != 0:
+        rel_tags.extend(db_tags)
+    if len(terms) != 0:
+        rel_tags.extend(terms)
+
+    print("rel_tags")
+    print(rel_tags)
+    mongo_ip = os.getenv("URL_MONGO_IP")
+    mongo_port = os.getenv("URL_MONGO_PORT")
+    mongo_url = "mongodb://" + mongo_ip + ":" + mongo_port
+    client = pymongo.MongoClient(mongo_url)
+    db = client.stackapis
+    collec_ques = db.questions
+    collec_ques.create_index('question_id')
+
+    # fetch questions
+    tot_ques = []
+    all_tags = []
+    entries = []
+    print("Fetching data from stackoverflow.. ")
+    SITE = StackAPI('stackoverflow')
+    # category = category.split()[0]
+    
+    # Fetching questions with only category name(e.g. latex) as tag
+    questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category, sort='votes', order='desc')
+    entries.extend(questions['items'])
+    
+    # Fetching questions with addition of rel_tags(obtained from quert/srt files)
+    for t in db_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+        
+    for t in rel_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+    
+    # inserting fetched data into mongodb
+    if len(entries) > 0:
+        for entry in entries:
+            collec_ques.update({'question_id': entry['question_id']}, entry, upsert=True)
+    print(str(len(entries)) + " questions fetched and inserted into mongodb")
+
+def get_questions_from_stack_sql(category, tutorial, query, terms, db_tags):
+    rel_tags = []
+    print("-----"*30,category,'\ntut',tutorial,'\nquery',query,'\nterms',terms,'\ndb_tags',db_tags)
+    # this part is run when the function is called from FAQ page with category and tutorial selected, here query=""
+    if len(db_tags) != 0:
+        rel_tags.extend(db_tags)
+    if len(terms) != 0:
+        rel_tags.extend(terms)
+
+    print("rel_tags")
+    print(rel_tags)
+    category_clean = re.sub(r'\d+','',category).strip(r".,: '")
+    print("category",category_clean)
+    # mongo_ip = os.getenv("URL_MONGO_IP")
+    # mongo_port = os.getenv("URL_MONGO_PORT")
+    # mongo_url = "mongodb://" + mongo_ip + ":" + mongo_port
+    # client = pymongo.MongoClient(mongo_url)
+    # db = client.stackapis
+    #collec_ques = db.questions
+    #collec_ques.create_index('question_id')
+    # fetch questions
+    tot_ques = []
+    all_tags = []
+    entries = []
+    print("Fetching data from stackoverflow.. ")
+    SITE = StackAPI('stackoverflow')
+    # category = category.split()[0]
+    # Fetching questions with only category name(e.g. latex) as tag
+    questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category_clean, sort='votes', order='desc')
+    print("questions :",questions['items'],'\n'*5)
+    entries.extend(questions['items'])
+    # Fetching questions with addition of rel_tags(obtained from quert/srt files)
+    for t in db_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category_clean+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+
+    for t in rel_tags:
+        questions = SITE.fetch('questions', fromdate=1456232494, min=20, tagged=category_clean+';'+t.lower(), sort='votes', order='desc')
+        entries.extend(questions['items'])
+
+    # inserting fetched data into mongodb
+    # if len(entries) > 0:
+    #     for entry in entries:
+    #         collec_ques.update({'question_id': entry['question_id']}, entry, upsert=True)
+    collec_ques, bool_created = StackQuestions.objects.get_or_create(tutorial=tutorial, category = category)
+    if bool_created:
+        collec_ques.stackdata = entries
+        collec_ques.save()
+
+    print(str(len(entries)) + " questions fetched and inserted into mongodb")
+
+
+def ajax_fetch_questions(request):
+    if request.method == 'POST':
+        category = request.POST['category']
+        tutorial = request.POST['tutorial']
+        td_rec = TutorialDetails.objects.using('spoken').filter(tutorial=tutorial, foss__foss=category).order_by('level', 'order')
+        cat = td_rec[0].foss
+        td = td_rec[0]
+        db_tags = TutorialCommonContent.objects.using('spoken').filter(tutorial_detail=td.pk)
+        db_tags = db_tags[0].keyword.replace(".", "").split(", ")
+        print("list of db_tags >>>>>>>>>>>>")
+        print(db_tags)
+        filename = settings.VIDEO_PATH + str(cat.pk) + '/' + str(td.pk) + '/' + tutorial.replace(' ', '-') + '-English.srt'
+        topic_keys = extract_keywords(filename)
+        print("topic_keys >>>>>>>>>>>>>")
+        print(topic_keys)
+
+    ''' Scraping stackoverflow to get questions and store in mongodb '''
+    #get_questions_from_stack(category.lower(), tutorial.lower(), '', topic_keys, db_tags)
+    get_questions_from_stack_sql(category.lower(), tutorial.lower(), '', topic_keys, db_tags)
+    return HttpResponse("Successfully fetched data from stackoverflow")
+
+def get_questions_from_db(topic_keys, db_tags, category, tutorial):
+    rel_tags = []
+    if len(db_tags) != 0:
+        rel_tags.extend(db_tags)
+    if len(topic_keys) != 0:
+        rel_tags.extend(topic_keys)
+    print("rel_tags")
+    print(rel_tags)
+    # mongo_ip = os.getenv("URL_MONGO_IP")
+    # mongo_port = os.getenv("URL_MONGO_PORT")
+    # mongo_url = "mongodb://" + mongo_ip + ":" + mongo_port
+    # client = pymongo.MongoClient(mongo_url)
+    # db = client.stackapi
+    # collec_ques = db.questions
+
+    questions = []
+    tags = []
+    q_ids = []
+    for t in rel_tags:
+        #items = collec_ques.find({"tags": t.lower()})
+        try:
+            items = StackQuestions.objects.get(category = category, tutorial=tutorial)
+            #print("Fetched " + str(items.count()) + " questions for tag: " + t.lower())
+            for item in items.stackdata:
+                tags.extend(item['tags'])
+                if item['question_id'] not in q_ids:
+                    acc_ans = ""
+                    if 'accepted_answer_id' in item.keys():
+                        acc_ans = str(item['link']) + "#" + str(item['accepted_answer_id'])
+                    q_ids.append(item['question_id'])
+                    q = {'title': item['title'],
+                    'uid': item['question_id'], 'body': item['link'],
+                    'tags': item['tags'], 'answer_count': item['answer_count'],
+                    'views': item['view_count'], 'score': item['score'],
+                    'owner': item['owner'], 'accepted_answer': acc_ans}
+                    questions.append(q)
+        except StackQuestions.DoesNotExist:
+            get_questions_from_stack_sql(category, tutorial, '', topic_keys, db_tags)
+    return questions, tags
+
+def ajax_faq_questions(request):
+    if request.method == 'POST':
+        category = request.POST['category']
+        tutorial = request.POST['tutorial']
+        td_rec = TutorialDetails.objects.using('spoken').filter(tutorial=tutorial, foss__foss=category).order_by('level', 'order')
+        cat = td_rec[0].foss
+        td = td_rec[0]
+        db_tags = TutorialCommonContent.objects.using('spoken').filter(tutorial_detail=td.pk)
+        db_tags = db_tags[0].keyword.replace(".", "").split(", ")
+        # db_tags.append(category.split()[0])
+        db_tags.append(category)
+        print("list of db_tags >>>>>>")
+        print(db_tags)
+        filename = settings.VIDEO_PATH + str(cat.pk) + '/' + str(td.pk) + '/' + tutorial.replace(' ', '-') + '-English.srt'
+        topic_keys = extract_keywords(filename)
+        print("topic_keys >>>>>>")
+        print(topic_keys)
+
+        ''' Fetching questions from mongodb database'''
+        #questions, tags = get_questions_from_db(topic_keys, db_tags)
+        questions, tags = get_questions_from_db(topic_keys, db_tags, category.lower(), tutorial.lower())
+        print(str(len(questions)) + " relevant questions and tags are ")
+        print(set(tags))
+        
+        context = {
+            'questions': questions,
+            'tags': tags
+        }
+        return render(request, 'website/templates/ajax_faq_questions.html', context)
+
 
 def ajax_similar_questions(request):
     if request.method == 'POST':
@@ -610,6 +827,10 @@ def ajax_hide_question(request):
 def ajax_keyword_search(request):
     if request.method == "POST":
         key = request.POST['key']
+        
+        titles = get_relevant_questions(key)
+        print("questions are")
+        print(titles)
         questions = Question.objects.filter(
             Q(title__icontains=key) | Q(category__icontains=key) |
             Q(tutorial__icontains=key) | Q(body__icontains=key), status=1
@@ -626,7 +847,8 @@ def ajax_keyword_search(request):
         except EmptyPage:
             questions = paginator.page(paginator.num_pages)
         context = {
-            'questions': questions
+            'questions': questions,
+            'titles': titles
         }
         return render(request, 'website/templates/ajax-keyword-search.html', context)