Spam and similar question

.gitignore

@@ -7,6 +7,8 @@ forums/local_settings.py
 static/CACHE/*
 *.py[cod]
 robots.txt
+cuss.csv
+stopwords_English.csv
 # C extensions
 *.so
 

static/website/js/custom.js

@@ -3,6 +3,7 @@ $(document).ready(function() {
     $tutorial = $("#id_tutorial");
     $minute_range = $("#id_minute_range");
     $second_range = $("#id_second_range");
+    $title = $("#id_title")
 		var tutorial = $tutorial.val();
 		var category = $category.val();
 
@@ -98,15 +99,16 @@ $(document).ready(function() {
         }
     });
 
-    $second_range.change(function() {
+    $title.change(function() {
         $.ajax({
             url: "/ajax-similar-questions/",
             type: "POST",
             data: {
-                category: $category.val(),
-                tutorial: $tutorial.val(),
-                minute_range: $minute_range.val(),
-                second_range: $second_range.val()
+                category        : $category.val(),
+                tutorial        : $tutorial.val(),
+                minute_range    : $minute_range.val(),
+                second_range    : $second_range.val(),
+                title           : $title.val(),
             },
             dataType: "html",
             success: function(data) {

static/website/templates/new-question.html

@@ -31,23 +31,42 @@ <h4>
         <div class="col-lg-2 col-md-2 col-sm-2">
             {% render_field form.second_range class+="form-control" disabled="disabled" %} 
         </div>
-        <div class="col-lg-2 col-md-2 col-sm-2">
-            <small><strong>
-            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
-                0 similar questions
-            </a>
-            </strong></small>
-        </div>
+
     </div>
     <hr>
+
+    {% if spam %}
+    <div id="spam" class="alert alert-danger alert-dismissible  show" style="color: white;font-size: 95%;font-family: sans serif;font-weight: 400;" role="alert">
+<button type="button" class="close" data-dismiss="alert" aria-label="Close">
+  <span aria-hidden="true">&times;</span>
+</button>
+    {{ spam }}
+<br><br><a href="/"> Back to Home</a>
 
+<meta http-equiv="refresh" content="5" />
+</div>
+    {% endif %}
     <p>Please enter your question details.</p>
+
     <div class="row">
         <div class="col-lg-12">
             <div class="form-group">
-                <label for="id_title">Title:</label>
+                 <div >
+                <label for="id_title">Title:
+
+
+            <strong>
+            <a id="similar-link" data-toggle="modal" data-target="#similarModal" href="#">
+                0 similar questions
+            </a>
+            </strong>
+
+                </label></div>
                 {% render_field form.title class+="form-control" %} 
             </div>
+
+
+            <div id="text"></div>
             <div class="form-group">
                 <label for="id_body">Question:</label>
                 {% render_field form.body class+="form-control" %}
@@ -59,7 +78,7 @@ <h4>
     </form>
 
     <!-- Modal -->
-    <div class="modal fade" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
+    <div class="modal" id="similarModal" tabindex="-1" role="dialog" aria-labelledby="similarModalLabel" aria-hidden="true">
       <div class="modal-dialog">
         <div class="modal-content">
           <div class="modal-header">
@@ -85,5 +104,7 @@ <h4 class="modal-title" id="myModalLabel">Similar Questions</h4>
 {% endblock %}
 
 {% block javascript %}
+
     <script src="{% static 'website/js/custom.js' %}"></script>
+
 {% endblock %}

website/forms.py

@@ -49,7 +49,7 @@ def __init__(self, *args, **kwargs):
             category = args[0]['category']
         if FossCategory.objects.filter(foss=category).exists():
             self.fields['category'].initial = category
-            tutorials = TutorialDetails.objects.using('spoken').filter(foss__foss=category)
+            tutorials = TutorialDetails.objects.using('spoken').filter(foss__foss=category).order_by('level', 'order')
             for tutorial in tutorials:
                 tutorial_choices += ((tutorial.tutorial, tutorial.tutorial),)
             self.fields['tutorial'] = forms.CharField(widget=forms.Select(choices=tutorial_choices))

website/spam_checker.py

@@ -0,0 +1,71 @@
+#from fuzzywuzzy import fuzz, process
+from website.models import Question, Answer, Notification, AnswerComment
+from spoken_auth.models import FossCategory, TutorialDetails
+from os.path import dirname, curdir, realpath
+import re, csv
+def existing_question(content, category, tutorial):
+    foss_id = FossCategory.objects.filter(foss = category)    
+    tutorial_detail = TutorialDetails.objects.filter(tutorial = tutorial)
+    keywords = TutorialCommonContents.objects.get(tutorial_detail = tutorial_detail)
+    foss_name = slugify_me(str(category))
+    tutorial = slugify_me(str(tutorial))
+    user_ques_parsed = remove_tags(parse_html(str(content)))
+    #user_ques_formatted = vectorizer.fit_transform(user_ques_parsed)
+    questions = Question.objects.filter(category=foss_name, tutorial=tutorial)
+    parsed_question = []
+    log =[]
+    for question in questions:
+
+        this_question = remove_tags(parse_html(str(question.body)))
+        if len(this_question)>4:
+            parsed_question.append(this_question)
+            log.append((this_question,question.id,fuzz.token_sort_ratio(user_ques_parsed, this_question)))
+            #formatted_question = vectorizer.fit_transform(parsed_question)
+            #print "->",process.extract(user_ques_parsed, this_question)
+    highest = process.extract(user_ques_parsed, parsed_question)
+
+
+def slugify_me(text):
+    data = text.strip().replace(' ','-')
+    return data
+
+def parse_html(text):
+    soup = BeautifulSoup(text, 'html.parser')
+    return soup.get_text()
+
+def takeThird(elem):
+    return elem[2]
+
+import re
+def striphtml(data):
+    p = re.compile(r'<.*?>')
+    return p.sub('', data)
+
+
+def check_for_cuss(text):
+    #text = striphtml(text)
+    dir_path = dirname(realpath(__file__))
+    filepath = dir_path+'/'+'cuss.csv'
+    body = re.split(', | ',str(text).lower().strip())
+    with open(filepath,'r') as csvfile:
+        spamreader = csv.reader(csvfile, delimiter=',')
+        for spam in spamreader:
+            for word in body:
+                if word in spam:
+                    return True
+            return False
+
+def remove_stop_words(text):
+    #text = striphtml(text)
+    title_words = re.split(', | ',str(text).lower())
+    print("My words are :",title_words)
+    important_words = []
+    dir_path = dirname(realpath(__file__))
+    filepath = dir_path+'/'+'stopwords_English.csv'
+    with open(filepath,'r') as csvfile:
+        stop_words = csv.reader(csvfile, delimiter=',')
+        for a_word in title_words:
+            if a_word not in stop_words :
+                important_words.append(a_word)
+    print("important_words : ",important_words)
+    return important_words

website/views.py

@@ -4,6 +4,7 @@
 from django.shortcuts import render, get_object_or_404
 from django.template.context_processors import csrf
 from django.contrib.auth.decorators import login_required
+from django.contrib import messages
 from django.db.models import Q
 from django.core.mail import EmailMultiAlternatives
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
@@ -19,7 +20,8 @@
 from .sortable import SortableHeader, get_sorted_list, get_field_index
 from django.db.models import Count
 
-
+#from spam_checker import check_for_cuss, remove_stop_words
+from . import spam_checker as sc
 User = get_user_model()
 categories = []
 trs = TutorialResources.objects.filter(Q(status=1) | Q(status=2),tutorial_detail__foss__show_on_homepage__lt=2, language__name='English')
@@ -288,9 +290,97 @@ def filter(request, category=None, tutorial=None, minute_range=None, second_rang
 def new_question(request):
     context = {}
     if request.method == 'POST':
-        form = NewQuestionForm(request.POST)
-        if form.is_valid():
+        form = NewQuestionForm(request.POST)        
+        spam_flag = False
+        if form.is_valid():            
             cleaned_data = form.cleaned_data
+            category = request.POST.get('category', None)
+            selected_tutorial = request.POST.get('tutorial', None)
+            tutorial = request.POST.get('tutorial', None)
+            content = request.POST['body']
+            title = request.POST['title'] 
+            minute_range = request.POST.get('minute_range', None) 
+            second_range = request.POST.get('second_range', None)
+            body = cleaned_data['body'].encode('unicode_escape')
+            context['tutorial'] = tutorial
+            #context['minute_range'] = minute_range            
+            #context['second_range'] = second_range
+            #context['body'] = content.lstrip().rstrip()
+            #context['title2'] = title
+            context['form'] = form
+            context['category'] = category
+            repeated_question = Question.objects.filter(category=category.replace(' ', '-'),
+                tutorial=selected_tutorial.replace(' ', '-'), title=cleaned_data['title'], body=body)
+            if repeated_question.exists():
+                display_message = "The exact same question was asked before hence has not been saved."
+                context['spam'] = display_message
+                context['questions'] = repeated_question
+                return HttpResponseRedirect("/")
+            else:
+                question = Question()
+                if sc.check_for_cuss(cleaned_data['body'].encode('unicode_escape').lower()):                              
+                    display_message = "You have entered a word which is either harmful or inappropriate according to our system.\
+                    Your question has been added for Admin Review."
+                    context['spam'] = display_message                
+                    question.status = 0
+                    spam_flag = True
+
+                question.uid = request.user.id
+                question.category = cleaned_data['category'].replace(' ', '-')
+                question.tutorial = cleaned_data['tutorial'].replace(' ', '-')
+                question.minute_range = minute_range
+                question.second_range = second_range
+                question.title = cleaned_data['title']
+                question.body = body
+                question.views = 1
+                question.save()
+
+
+                # Sending email when a new question is asked
+
+                message = """
+                    The following new question has been posted in the Spoken Tutorial Forum: <br>
+                    Title: <b>{0}</b><br>
+                    Category: <b>{1}</b><br>
+                    Tutorial: <b>{2}</b><br>
+                    Link: <a href="{3}">{3}</a><br>
+                    Question: <b>{4}</b><br>
+                """.format(
+                    question.title,
+                    question.category,
+                    question.tutorial,
+                    'http://forums.spoken-tutorial.org/question/' + str(question.id),
+                    question.body
+                )
+                if spam_flag :
+                    subject = 'New Forum Question in Spams'
+                    #mail_to = 'admin@spoken-tutorial.org,nancyvarkey.iitb@gmail.com'
+                    mail_to = 'adhikarysaurabh@gmail.com'
+                    forums_mail(mail_to, subject, message)
+                    # email = EmailMultiAlternatives(
+                    #     subject, '', 'forums',
+                    #     ['admin@spoken-tutorial.org', 'nancyvarkey.iitb@gmail.com'],
+                    #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                    # )
+                    # email.attach_alternative(message, "text/html")
+                    # email.send(fail_silently=True)
+                    # End of email send
+                    return render(request, 'website/templates/new-question.html', context)
+                else:
+                    subject = 'New Forum Question'
+                    #mail_to = 'team@spoken-tutorial.org,team@fossee.in'
+                    mail_to = 'adhikarysaurabh@gmail.com'
+                    forums_mail(mail_to, subject, message)
+                    # email = EmailMultiAlternatives(
+                    #     subject, '', 'forums',
+                    #     ['team@spoken-tutorial.org', 'team@fossee.in'],
+                    #     headers={"Content-type": "text/html;charset=iso-8859-1"}
+                    # )
+                    # email.attach_alternative(message, "text/html")
+                    # email.send(fail_silently=True)
+                    # End of email send
+                    return HttpResponseRedirect("/")
+
             question = Question()
             question.uid = request.user.id
             question.category = cleaned_data['category'].replace(' ', '-')
@@ -338,7 +428,9 @@ def new_question(request):
         form = NewQuestionForm(category=category, tutorial=tutorial,
                                minute_range=minute_range, second_range=second_range)
         context['category'] = category
-
+        context['tut'] = tutorial
+        context['minute_range'] = minute_range            
+        context['second_range'] = second_range
     context['form'] = form
     context.update(csrf(request))
     return render(request, 'website/templates/new-question.html', context)
@@ -536,14 +628,21 @@ def ajax_similar_questions(request):
     if request.method == 'POST':
         category = request.POST['category']
         tutorial = request.POST['tutorial']
-        # minute_range = request.POST['minute_range']
-        # second_range = request.POST['second_range']
-
-        # add more filtering when the forum grows
-        questions = Question.objects.filter(category=category).filter(tutorial=tutorial)
+        category = category.replace(' ','-')
+        tutorial = tutorial.replace(' ','-')
+        title = request.POST['title']
+        MIN_LENGTH_OF_TITLE = 3
+        questions = Question.objects.none()
+        if len(title) > MIN_LENGTH_OF_TITLE:
+            important_words = sc.remove_stop_words(title.lower().encode('unicode_escape'))
+            # add more filtering when the forum grows
+            for a_word in important_words:
+                required_ques = Question.objects.filter(category=category,tutorial=tutorial,
+                    title__icontains=a_word, status=1)
+                questions = questions | required_ques
         context = {
-            'questions': questions
-        }
+                'questions': questions
+            }
         return render(request, 'website/templates/ajax-similar-questions.html', context)