-
Notifications
You must be signed in to change notification settings - Fork 12
/
adminviews.py
executable file
·8352 lines (6725 loc) · 408 KB
/
adminviews.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import json
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.db.models import Q, F, ExpressionWrapper, IntegerField, Count
from django.db.models import CharField, TextField, Value as V
from django.db.models import OuterRef, Subquery
from django.db.models.query import QuerySet
from django.db.models.functions import Concat
from django.db.models.fields import BooleanField, BooleanField
from django.db.models.sql.where import NothingNode, WhereNode
from django.http import HttpResponse, HttpResponseRedirect, QueryDict, JsonResponse
from django.template import RequestContext
from django.urls import reverse_lazy
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
from django.core.paginator import Paginator
from django.utils.translation import override, gettext_lazy as _, activate
from django.utils.html import escape
from django.forms.fields import ChoiceField
from django.shortcuts import *
from django.contrib import messages
from django.contrib.sites.models import Site
from django.template.loader import render_to_string
from signbank.dictionary.templatetags.field_choice import get_field_choice
from django.contrib.auth.models import User, Group
import csv
import operator
import re
import xml.etree.ElementTree as ET
from xml.dom import minidom
import datetime as DT
from guardian.core import ObjectPermissionChecker
from guardian.shortcuts import get_objects_for_user
from signbank.dictionary.models import *
from signbank.dictionary.forms import *
from signbank.feedback.models import *
from signbank.video.forms import VideoUploadForObjectForm
from tagging.models import Tag, TaggedItem
from signbank.settings.base import ECV_FILE,EARLIEST_GLOSS_CREATION_DATE, FIELDS, SEPARATE_ENGLISH_IDGLOSS_FIELD, \
LANGUAGE_CODE, ECV_SETTINGS, URL, LANGUAGE_CODE_MAP, LANGUAGES, LANGUAGES_LANGUAGE_CODE_3CHAR
from signbank.settings import server_specific
from signbank.settings.server_specific import *
from signbank.dictionary.translate_choice_list import machine_value_to_translated_human_value, \
choicelist_queryset_to_translated_dict, choicelist_queryset_to_machine_value_dict, choicelist_queryset_to_colors, \
choicelist_queryset_to_field_colors
from signbank.dictionary.field_choices import get_static_choice_lists, get_frequencies_for_category, category_to_fields, \
fields_to_categories, fields_to_fieldcategory_dict
from signbank.dictionary.forms import GlossSearchForm, MorphemeSearchForm
from django.forms import TypedMultipleChoiceField, ChoiceField
from signbank.dictionary.update import upload_metadata
from signbank.tools import get_selected_datasets_for_user, write_ecv_file_for_dataset, write_csv_for_handshapes, \
construct_scrollbar, write_csv_for_minimalpairs, get_dataset_languages, get_datasets_with_public_glosses, \
searchform_panels, map_search_results_to_gloss_list, \
get_interface_language_and_default_language_codes
from signbank.csv_interface import sense_translations_for_language, sense_examplesentences_for_language
from signbank.dictionary.update_senses_mapping import delete_empty_senses
from signbank.dictionary.consistency_senses import consistent_senses, check_consistency_senses, \
reorder_sensetranslations, reorder_senses
from signbank.query_parameters import convert_query_parameters_to_filter, pretty_print_query_fields, pretty_print_query_values, \
query_parameters_this_gloss, apply_language_filters_to_results
from signbank.search_history import available_query_parameters_in_search_history, languages_in_query, display_parameters, \
get_query_parameters, save_query_parameters, fieldnames_from_query_parameters
from signbank.frequency import import_corpus_speakers, configure_corpus_documents_for_dataset, update_corpus_counts, \
speaker_identifiers_contain_dataset_acronym, get_names_of_updated_eaf_files, update_corpus_document_counts, \
dictionary_speakers_to_documents, document_has_been_updated, document_to_number_of_glosses, \
document_to_glosses, get_corpus_speakers, remove_document_from_corpus, document_identifiers_from_paths, \
eaf_file_from_paths, documents_paths_dictionary
from signbank.dictionary.frequency_display import collect_speaker_age_data, collect_variants_data, collect_variants_age_range_data, \
collect_variants_age_sex_raw_percentage
from signbank.dictionary.senses_display import (sensetranslations_per_language, sensetranslations_per_language_list,
sensetranslations_per_language_dict,
senses_translations_per_language_list, senses_sentences_per_language_list)
def order_queryset_by_sort_order(get, qs, queryset_language_codes):
"""Change the sort-order of the query set, depending on the form field [sortOrder]
This function is used both by GlossListView as well as by MorphemeListView.
The value of [sortOrder] is 'lemma__lemmaidglosstranslation__text' by default.
[sortOrder] is a hidden field inside the "adminsearch" html form in the template admin_gloss_list.html
Its value is changed by clicking the up/down buttons in the second row of the search result table
"""
# Helper: order a queryset on field [sOrder], which is a number from a list of tuples named [sListName]
def order_queryset_by_tuple_list(qs, sOrder, sListName, bReversed):
"""Order a queryset on field [sOrder], which is a number from a list of tuples named [sListName]"""
# Get a list of tuples for this sort-order
tpList = list(FieldChoice.objects.filter(field=sListName).values_list('machine_value', 'name'))
# Determine sort order: ascending is default
if (sOrder[0:1] == '-'):
# A starting '-' sign means: descending order
sOrder = sOrder[1:]
def lambda_sort_tuple(x, bReversed):
# Order by the string-values in the tuple list
getattr_sOrder = getattr(x, sOrder)
if getattr_sOrder is None:
# if the field is not set, use the machine value 0 choice
return (True, dict(tpList)[0])
elif getattr_sOrder.machine_value in [0,1]:
return (True, dict(tpList)[getattr_sOrder.machine_value])
else:
return(bReversed, dict(tpList)[getattr(x, sOrder).machine_value])
return sorted(qs, key=lambda x: lambda_sort_tuple(x, bReversed), reverse=bReversed)
def order_queryset_by_annotationidglosstranslation(qs, sOrder):
language_code_2char = sOrder[-2:]
sOrderAsc = sOrder
if (sOrder[0:1] == '-'):
# A starting '-' sign means: descending order
sOrderAsc = sOrder[1:]
annotationidglosstranslation = AnnotationIdglossTranslation.objects.filter(gloss=OuterRef('pk')).filter(language__language_code_2char__iexact=language_code_2char).distinct()
qs = qs.annotate(**{sOrderAsc: Subquery(annotationidglosstranslation.values('text')[:1])}).order_by(sOrder)
return qs
def order_queryset_by_lemmaidglosstranslation(qs, sOrder):
language_code_2char = sOrder[-2:]
sOrderAsc = sOrder
if (sOrder[0:1] == '-'):
# A starting '-' sign means: descending order
sOrderAsc = sOrder[1:]
lemmaidglosstranslation = LemmaIdglossTranslation.objects.filter(lemma=OuterRef('lemma'), language__language_code_2char__iexact=language_code_2char)
qs = qs.annotate(**{sOrderAsc: Subquery(lemmaidglosstranslation.values('text')[:1])}).order_by(sOrder)
return qs
def order_queryset_by_translation(qs, sOrder):
language_code_2char = sOrder[-2:]
sOrderAsc = sOrder
if (sOrder[0:1] == '-'):
# A starting '-' sign means: descending order
sOrderAsc = sOrder[1:]
translations = Translation.objects.filter(sensetranslation__sense__glosssense__gloss=OuterRef('pk')).filter(language__language_code_2char__iexact=language_code_2char)
qs = qs.annotate(**{sOrderAsc: Subquery(translations.values('translation__text')[:1])}).order_by(sOrder)
return qs
# Set the default sort order
default_sort_order = True
bReversed = False
bText = True
# See if the form contains any sort-order information
if ('sortOrder' in get and get['sortOrder'] != ''):
# Take the user-indicated sort order
sOrder = get['sortOrder']
default_sort_order = False
if (sOrder[0:1] == '-'):
# A starting '-' sign means: descending order
bReversed = True
else:
# this is important for the query
# this is used by default_sort_order for the filtering out of annotations that start with symbols
# so that they can be moved to the end of the results
sOrder = 'annotationidglosstranslation__text' # Default sort order if nothing is specified
# The ordering method depends on the kind of field:
# (1) text fields are ordered straightforwardly
# (2) fields made from a choice_list need special treatment
if (sOrder.endswith('handedness')):
bText = False
ordered = order_queryset_by_tuple_list(qs, sOrder, "Handedness", bReversed)
elif (sOrder.endswith('domhndsh') or sOrder.endswith('subhndsh')):
bText = False
ordered = order_queryset_by_tuple_list(qs, sOrder, "Handshape", bReversed)
elif (sOrder.endswith('locprim')):
bText = False
ordered = order_queryset_by_tuple_list(qs, sOrder, "Location", bReversed)
elif sOrder.startswith("annotationidglosstranslation_order_") or sOrder.startswith("-annotationidglosstranslation_order_"):
ordered = order_queryset_by_annotationidglosstranslation(qs, sOrder)
elif sOrder.startswith("lemmaidglosstranslation_order_") or sOrder.startswith("-lemmaidglosstranslation_order_"):
ordered = order_queryset_by_lemmaidglosstranslation(qs, sOrder)
elif sOrder.startswith("translation_") or sOrder.startswith("-translation_"):
ordered = order_queryset_by_translation(qs, sOrder)
else:
# Use straightforward ordering on field [sOrder]
if default_sort_order:
lang_attr_name = settings.DEFAULT_KEYWORDS_LANGUAGE['language_code_2char']
sort_language = 'annotationidglosstranslation__language__language_code_2char'
if len(queryset_language_codes) == 0:
ordered = qs
else:
if lang_attr_name not in queryset_language_codes:
lang_attr_name = queryset_language_codes[0]
qs_empty = qs.filter(**{sOrder+'__isnull': True})
qs_letters = qs.filter(**{sOrder+'__regex':r'^[a-zA-Z]', sort_language:lang_attr_name})
qs_special = qs.filter(**{sOrder+'__regex':r'^[^a-zA-Z]', sort_language:lang_attr_name})
# sort_key = sOrder
# # Using the order_by here results in duplicating the objects!
ordered = list(qs_letters) #.order_by(sort_key))
ordered += list(qs_special) #.order_by(sort_key))
ordered += list(qs_empty)
else:
ordered = qs
if bReversed and bText:
ordered.reverse()
# return the ordered list
return ordered
class GlossListView(ListView):
model = Gloss
paginate_by = 100
only_export_ecv = False #Used to call the 'export ecv' functionality of this view without the need for an extra GET parameter
search_type = 'sign'
view_type = 'gloss_list'
web_search = False
show_all = False
dataset_name = settings.DEFAULT_DATASET_ACRONYM
last_used_dataset = None
queryset_language_codes = []
query_parameters = dict()
search_form_data = QueryDict(mutable=True)
def get_template_names(self):
return ['dictionary/admin_gloss_list.html']
def get_context_data(self, **kwargs):
# Call the base implementation first to get a context
context = super(GlossListView, self).get_context_data(**kwargs)
if 'show_all' in self.kwargs.keys():
context['show_all'] = self.kwargs['show_all']
self.show_all = self.kwargs['show_all']
else:
context['show_all'] = self.show_all
# Retrieve the search_type,so that we know whether the search should be restricted to Gloss or not
if 'search_type' in self.request.GET:
self.search_type = self.request.GET['search_type']
if 'search' in self.request.GET:
context['menu_bar_search'] = self.request.GET['search']
if 'search_type' not in self.request.session.keys():
self.request.session['search_type'] = self.search_type
if 'view_type' in self.request.GET:
# user is adjusting the view, leave the rest of the context alone
self.view_type = self.request.GET['view_type']
context['view_type'] = self.view_type
if 'inWeb' in self.request.GET:
# user is searching for signs / morphemes visible to anonymous uers
self.web_search = self.request.GET['inWeb'] == '2'
elif not self.request.user.is_authenticated:
self.web_search = True
context['web_search'] = self.web_search
if self.request.user.is_authenticated:
selected_datasets = get_selected_datasets_for_user(self.request.user)
elif 'selected_datasets' in self.request.session.keys():
selected_datasets = Dataset.objects.filter(acronym__in=self.request.session['selected_datasets'])
else:
selected_datasets = Dataset.objects.filter(acronym=settings.DEFAULT_DATASET_ACRONYM)
dataset_languages = get_dataset_languages(selected_datasets)
context['dataset_languages'] = dataset_languages
# the following is needed by javascript in the case only one dataset is available
# in order not to compute dynamically in the template
dataset_languages_abbreviations = []
for ds in selected_datasets:
for sdl in ds.translation_languages.all():
if sdl.language_code_2char not in dataset_languages_abbreviations:
dataset_languages_abbreviations.append(sdl.language_code_2char)
js_dataset_languages = ','.join(dataset_languages_abbreviations)
context['js_dataset_languages'] = js_dataset_languages
default_dataset_acronym = settings.DEFAULT_DATASET_ACRONYM
default_dataset = Dataset.objects.get(acronym=default_dataset_acronym)
for lang in dataset_languages:
if lang.language_code_2char not in self.queryset_language_codes:
self.queryset_language_codes.append(lang.language_code_2char)
if self.queryset_language_codes is None:
self.queryset_language_codes = [ default_dataset.default_language.language_code_2char ]
if len(selected_datasets) == 1:
self.last_used_dataset = selected_datasets.first().acronym
elif 'last_used_dataset' in self.request.session.keys():
self.last_used_dataset = self.request.session['last_used_dataset']
context['last_used_dataset'] = self.last_used_dataset
selected_datasets_signlanguage = list(SignLanguage.objects.filter(dataset__in=selected_datasets))
sign_languages = []
for sl in selected_datasets_signlanguage:
if (str(sl.id),sl.name) not in sign_languages:
sign_languages.append((str(sl.id), sl.name))
selected_datasets_dialects = Dialect.objects.filter(signlanguage__in=selected_datasets_signlanguage)\
.prefetch_related('signlanguage').distinct()
dialects = []
for dl in selected_datasets_dialects:
dialect_name = dl.signlanguage.name + "/" + dl.name
dialects.append((str(dl.id),dialect_name))
if not self.show_all and ('query_parameters' in self.request.session.keys()
and self.request.session['query_parameters'] not in ['', '{}']):
# if the query parameters are available, convert them to a dictionary
session_query_parameters = self.request.session['query_parameters']
self.query_parameters = json.loads(session_query_parameters)
search_form = GlossSearchForm(self.request.GET, languages=dataset_languages, sign_languages=sign_languages,
dialects=dialects)
context['query_parameters'] = json.dumps(self.query_parameters)
query_parameters_keys = list(self.query_parameters.keys())
context['query_parameters_keys'] = json.dumps(query_parameters_keys)
# other parameters are in the GlossSearchForm in the template that are not initialised via multiselect or language fields
# plus semantics and phonology fields with text types
other_parameters = ['sortOrder'] + \
settings.SEARCH_BY['publication'] + \
settings.SEARCH_BY['relations'] + \
settings.SEARCH_BY['morpheme'] + \
settings.FIELDS['phonology'] + \
settings.FIELDS['semantics']
fieldnames = FIELDS['main']+FIELDS['phonology']+FIELDS['semantics']+['inWeb', 'isNew']
fields_with_choices = fields_to_fieldcategory_dict()
multiple_select_gloss_fields = [fieldname for fieldname in fieldnames if fieldname in fields_with_choices.keys()]
other_parameters_keys = [ key for key in other_parameters if key not in multiple_select_gloss_fields ]
context['other_parameters_keys'] = json.dumps(other_parameters_keys)
# This is needed to display the idgloss of the morpheme in Search by Morphology: Search for gloss with this as morpheme
# The id of the morpheme selected in the GlossSearchForm is kept in a hidden input field
# after selection from the lookahead list
if 'morpheme' in self.query_parameters.keys():
try:
morpheme_idgloss = Morpheme.objects.get(pk=self.query_parameters['morpheme']).idgloss
except ObjectDoesNotExist:
morpheme_idgloss = ''
else:
morpheme_idgloss = ''
context['morpheme_idgloss'] = morpheme_idgloss
gloss_search_field_prefix = "glosssearch_"
keyword_search_field_prefix = "keyword_"
lemma_search_field_prefix = "lemma_"
language_query_keys = []
for language in dataset_languages:
glosssearch_field_name = gloss_search_field_prefix + language.language_code_2char
language_query_keys.append(glosssearch_field_name)
lemma_field_name = lemma_search_field_prefix + language.language_code_2char
language_query_keys.append(lemma_field_name)
keyword_field_name = keyword_search_field_prefix + language.language_code_2char
language_query_keys.append(keyword_field_name)
context['language_query_keys'] = json.dumps(language_query_keys)
context['searchform'] = search_form
context['search_type'] = self.search_type
context['view_type'] = self.view_type
context['web_search'] = self.web_search
# If the menu bar search form was used, populate the search form with the query string
gloss_fields_to_populate = dict()
if 'search' in self.request.GET and self.request.GET['search'] != '':
val = self.request.GET['search']
from signbank.tools import strip_control_characters
val = strip_control_characters(val)
gloss_fields_to_populate['search'] = escape(val)
if 'translation' in self.request.GET and self.request.GET['translation'] != '':
val = self.request.GET['translation']
from signbank.tools import strip_control_characters
val = strip_control_characters(val)
gloss_fields_to_populate['translation'] = escape(val)
gloss_fields_to_populate_keys = list(gloss_fields_to_populate.keys())
context['gloss_fields_to_populate'] = json.dumps(gloss_fields_to_populate)
context['gloss_fields_to_populate_keys'] = gloss_fields_to_populate_keys
context['default_dataset_lang'] = dataset_languages.first().language_code_2char if dataset_languages else LANGUAGE_CODE
context['add_gloss_form'] = GlossCreateForm(self.request.GET, languages=dataset_languages, user=self.request.user, last_used_dataset=self.last_used_dataset)
if hasattr(settings, 'SHOW_DATASET_INTERFACE_OPTIONS'):
context['SHOW_DATASET_INTERFACE_OPTIONS'] = settings.SHOW_DATASET_INTERFACE_OPTIONS
else:
context['SHOW_DATASET_INTERFACE_OPTIONS'] = False
if hasattr(settings, 'SHOW_MORPHEME_SEARCH') and self.request.user.is_authenticated:
context['SHOW_MORPHEME_SEARCH'] = settings.SHOW_MORPHEME_SEARCH
else:
context['SHOW_MORPHEME_SEARCH'] = False
if hasattr(settings, 'GLOSS_LIST_DISPLAY_HEADER') and self.request.user.is_authenticated:
context['GLOSS_LIST_DISPLAY_HEADER'] = settings.GLOSS_LIST_DISPLAY_HEADER
else:
context['GLOSS_LIST_DISPLAY_HEADER'] = []
if hasattr(settings, 'SEARCH_BY') and 'publication' in settings.SEARCH_BY.keys() and self.request.user.is_authenticated:
context['search_by_publication_fields'] = searchform_panels(search_form, settings.SEARCH_BY['publication'])
else:
context['search_by_publication_fields'] = []
if hasattr(settings, 'SEARCH_BY') and 'relations' in settings.SEARCH_BY.keys() and self.request.user.is_authenticated:
context['search_by_relation_fields'] = searchform_panels(search_form, settings.SEARCH_BY['relations'])
else:
context['search_by_relation_fields'] = []
multiple_select_gloss_fields.append('definitionRole')
multiple_select_gloss_fields.append('hasComponentOfType')
context['MULTIPLE_SELECT_GLOSS_FIELDS'] = multiple_select_gloss_fields
fields_with_choices['definitionRole'] = 'NoteType'
fields_with_choices['hasComponentOfType'] = 'MorphologyType'
choices_colors = {}
for (fieldname, field_category) in fields_with_choices.items():
if field_category in CATEGORY_MODELS_MAPPING.keys():
field_choices = CATEGORY_MODELS_MAPPING[field_category].objects.all()
else:
field_choices = FieldChoice.objects.filter(field__iexact=field_category)
choices_colors[fieldname] = json.dumps(choicelist_queryset_to_field_colors(field_choices))
context['field_colors'] = choices_colors
if hasattr(settings, 'DISABLE_MOVING_THUMBNAILS_ABOVE_NR_OF_GLOSSES'):
context['DISABLE_MOVING_THUMBNAILS_ABOVE_NR_OF_GLOSSES'] = settings.DISABLE_MOVING_THUMBNAILS_ABOVE_NR_OF_GLOSSES
else:
context['DISABLE_MOVING_THUMBNAILS_ABOVE_NR_OF_GLOSSES'] = 0
context['input_names_fields_and_labels'] = {}
for topic in ['main','phonology','semantics']:
context['input_names_fields_and_labels'][topic] = []
for fieldname in settings.FIELDS[topic]:
if fieldname == 'derivHist' and not settings.USE_DERIVATIONHISTORY:
continue
# exclude the dependent fields for Handedness, Strong Hand, and Weak Hand for purposes of nested dependencies in Search form
if fieldname not in settings.HANDSHAPE_ETYMOLOGY_FIELDS + settings.HANDEDNESS_ARTICULATION_FIELDS:
field = search_form[fieldname]
label = field.label
context['input_names_fields_and_labels'][topic].append((fieldname,field,label))
context['input_names_fields_labels_handedness'] = []
field = search_form['weakdrop']
label = field.label
context['input_names_fields_labels_handedness'].append(('weakdrop', field, label))
field = search_form['weakprop']
label = field.label
context['input_names_fields_labels_handedness'].append(('weakprop',field,label))
context['input_names_fields_labels_domhndsh'] = []
field = search_form['domhndsh_letter']
label = field.label
context['input_names_fields_labels_domhndsh'].append(('domhndsh_letter',field,label))
field = search_form['domhndsh_number']
label = field.label
context['input_names_fields_labels_domhndsh'].append(('domhndsh_number',field,label))
context['input_names_fields_labels_subhndsh'] = []
field = search_form['subhndsh_letter']
label = field.label
context['input_names_fields_labels_subhndsh'].append(('subhndsh_letter',field,label))
field = search_form['subhndsh_number']
label = field.label
context['input_names_fields_labels_subhndsh'].append(('subhndsh_number',field,label))
context['default_dataset_lang'] = dataset_languages.first().language_code_2char if dataset_languages else LANGUAGE_CODE
context['lemma_create_field_prefix'] = LemmaCreateForm.lemma_create_field_prefix
# it is necessary to sort the object list by lemma_id in order for all glosses with the same lemma to be grouped
# correctly in the template
list_of_object_ids = [ g.id for g in self.object_list ]
glosses_ordered_by_lemma_id = Gloss.objects.filter(id__in=list_of_object_ids).order_by('lemma_id')
context['glosses_ordered_by_lemma_id'] = glosses_ordered_by_lemma_id
if self.search_type == 'sign' or not self.request.user.is_authenticated:
# Only count the none-morpheme glosses
# this branch is slower than the other one
context['glosscount'] = Gloss.none_morpheme_objects().select_related('lemma').select_related('dataset').filter(lemma__dataset__in=selected_datasets).count()
else:
context['glosscount'] = Gloss.objects.select_related('lemma').select_related('dataset').filter(lemma__dataset__in=selected_datasets).count() # Count the glosses + morphemes
context['page_number'] = context['page_obj'].number
context['objects_on_page'] = [ g.id for g in context['page_obj'].object_list ]
# this is needed to avoid crashing the browser if you go to the last page
# of an extremely long list and then go to Details on the objects
this_page_number = context['page_obj'].number
this_paginator = context['page_obj'].paginator
if len(self.object_list) > settings.MAX_SCROLL_BAR:
this_page = this_paginator.page(this_page_number)
if this_page.has_previous():
previous_objects = this_paginator.page(this_page_number - 1).object_list
else:
previous_objects = []
if this_page.has_next():
next_objects = this_paginator.page(this_page_number + 1).object_list
else:
next_objects = []
list_of_objects = previous_objects + list(context['page_obj'].object_list) + next_objects
else:
list_of_objects = self.object_list
# construct scroll bar
# the following retrieves language code for English (or DEFAULT LANGUAGE)
# so the sorting of the scroll bar matches the default sorting of the results in Gloss List View
(interface_language, interface_language_code,
default_language, default_language_code) = get_interface_language_and_default_language_codes(self.request)
dataset_display_languages = []
for lang in dataset_languages:
dataset_display_languages.append(lang.language_code_2char)
if interface_language_code in dataset_display_languages:
lang_attr_name = interface_language_code
else:
lang_attr_name = default_language_code
items = construct_scrollbar(list_of_objects, self.search_type, lang_attr_name)
self.request.session['search_results'] = items
if 'paginate_by' in self.request.GET:
context['paginate_by'] = int(self.request.GET.get('paginate_by'))
self.request.session['paginate_by'] = context['paginate_by']
else:
if 'paginate_by' in self.request.session.keys():
# restore any previous paginate setting for toggling between Lemma View and Gloss List View
# the session variable is needed when you return to the List View after looking at the Lemma View
context['paginate_by'] = self.request.session['paginate_by']
else:
context['paginate_by'] = self.paginate_by
column_headers = []
for fieldname in settings.GLOSS_LIST_DISPLAY_FIELDS:
field_label = Gloss.get_field(fieldname).verbose_name
column_headers.append((fieldname, field_label))
context['column_headers'] = column_headers
return context
def get_paginate_by(self, queryset):
"""
Paginate by specified value in querystring, or use default class property value.
"""
# Toelichting (Information about coding):
# Django generates a new context when one toggles to Lemma View.
# Lemma View uses a regroup on the object list and also uses the default paginate_by in self.
# If the user resets the paginate_by in Gloss List, this setup (session variable
# that's only retrieved for Gloss View) handles returning to the previous paginate_by.
# Because the Lemma View is sparsely populated, if the default pagination isn't used,
# there are pages without contents, since only Lemma's with more than one gloss are shown.
# We're essentially remembering the previous paginate_by for when the user
# toggles back to Gloss View after List View
if 'paginate_by' in self.request.GET:
paginate_by = int(self.request.GET.get('paginate_by'))
self.request.session['paginate_by'] = paginate_by
else:
if self.view_type == 'lemma_groups':
paginate_by = self.paginate_by
elif 'paginate_by' in self.request.session.keys():
# restore any previous paginate setting for toggling between Lemma View and Gloss List View
# the session variable is needed when you return to the List View after looking at the Lemma View
paginate_by = self.request.session['paginate_by']
else:
paginate_by = self.paginate_by
return paginate_by
def render_to_response(self, context):
# Look for a 'format=json' GET argument
if self.request.GET.get('format') == 'CSV':
# show_all is passed by the calling template
return self.render_to_csv_response({'show_all': self.request.GET.get('show_all')})
elif self.request.GET.get('export_ecv') == 'ECV' or self.only_export_ecv:
return self.render_to_ecv_export_response(context)
else:
return super(GlossListView, self).render_to_response(context)
def render_to_ecv_export_response(self, context):
# check that the user is logged in
if self.request.user.is_authenticated:
pass
else:
messages.add_message(self.request, messages.ERROR, _('Please login to use this functionality.'))
return HttpResponseRedirect(settings.PREFIX_URL + '/signs/search/')
# if the dataset is specified in the url parameters, set the dataset_name variable
get = self.request.GET
if 'dataset_name' in get:
self.dataset_name = get['dataset_name']
if self.dataset_name == '':
messages.add_message(self.request, messages.ERROR, _('Dataset name must be non-empty.'))
return HttpResponseRedirect(settings.PREFIX_URL + '/signs/search/')
try:
dataset_object = Dataset.objects.get(name=self.dataset_name)
except ObjectDoesNotExist:
messages.add_message(self.request, messages.ERROR, _('No dataset with that name found.'))
return HttpResponseRedirect(settings.PREFIX_URL + '/signs/search/')
# make sure the user can write to this dataset
import guardian
# from guardian.shortcuts import get_objects_for_user
user_change_datasets = guardian.shortcuts.get_objects_for_user(self.request.user, 'change_dataset', Dataset)
if user_change_datasets and dataset_object in user_change_datasets:
pass
else:
messages.add_message(self.request, messages.ERROR, _('No permission to export dataset.'))
return HttpResponseRedirect(settings.PREFIX_URL + '/signs/search/')
# if we get to here, the user is authenticated and has permission to export the dataset
success, ecv_file = write_ecv_file_for_dataset(self.dataset_name)
if success:
messages.add_message(self.request, messages.INFO, _('ECV successfully updated.'))
else:
messages.add_message(self.request, messages.INFO, _('No ECV created for dataset.'))
return HttpResponseRedirect(settings.PREFIX_URL + '/signs/search/')
# noinspection PyInterpreter,PyInterpreter
def render_to_csv_response(self, context):
if not self.request.user.has_perm('dictionary.export_csv'):
raise PermissionDenied
if 'show_all' in context.keys():
show_all = context['show_all']
else:
show_all = False
# Create the HttpResponse object with the appropriate CSV header.
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="dictionary-export.csv"'
fieldnames = FIELDS['main']+FIELDS['phonology']+FIELDS['semantics']+FIELDS['frequency']+['inWeb', 'isNew']
fields = [Gloss.get_field(fname) for fname in fieldnames if fname in Gloss.get_field_names()]
selected_datasets = get_selected_datasets_for_user(self.request.user)
dataset_languages = get_dataset_languages(selected_datasets)
lang_attr_name = 'name_' + DEFAULT_KEYWORDS_LANGUAGE['language_code_2char']
annotationidglosstranslation_fields = ["Annotation ID Gloss" + " (" + getattr(language, lang_attr_name) + ")"
for language in dataset_languages]
lemmaidglosstranslation_fields = ["Lemma ID Gloss" + " (" + getattr(language, lang_attr_name) + ")"
for language in dataset_languages]
keyword_fields = ["Senses" + " (" + getattr(language, lang_attr_name) + ")"
for language in dataset_languages]
sentence_fields = ["Example Sentences" + " (" + getattr(language, lang_attr_name) + ")"
for language in dataset_languages]
writer = csv.writer(response)
# CSV should be the first language in the settings
activate(LANGUAGES[0][0])
header = ['Signbank ID', 'Dataset'] + lemmaidglosstranslation_fields + annotationidglosstranslation_fields \
+ keyword_fields + sentence_fields + [f.verbose_name.encode('ascii','ignore').decode() for f in fields]
for extra_column in ['SignLanguages','Dialects', 'Sequential Morphology', 'Simultaneous Morphology', 'Blend Morphology',
'Relations to other signs','Relations to foreign signs', 'Tags', 'Notes']:
header.append(extra_column)
writer.writerow(header)
if self.object_list:
query_set = self.object_list
else:
query_set = self.get_queryset()
# for some reason when show_all has been selected, the object list has become a list instead of a QuerySet
# it was also missing elements
# in order to simply debug print statements, it's converted to a list here to make sure it always has the same type
if isinstance(query_set, QuerySet):
query_set = list(query_set)
for gloss in query_set:
row = [str(gloss.pk), gloss.lemma.dataset.acronym]
for language in dataset_languages:
lemmaidglosstranslations = gloss.lemma.lemmaidglosstranslation_set.filter(language=language)
if lemmaidglosstranslations and len(lemmaidglosstranslations) == 1:
# get rid of any invisible characters at the end such as \t
lemmatranslation = lemmaidglosstranslations.first().text.strip()
row.append(lemmatranslation)
else:
row.append("")
for language in dataset_languages:
annotationidglosstranslations = gloss.annotationidglosstranslation_set.filter(language=language)
if annotationidglosstranslations and len(annotationidglosstranslations) == 1:
# get rid of any invisible characters at the end such as \t
annotation = annotationidglosstranslations.first().text.strip()
row.append(annotation)
else:
row.append("")
# Put senses (keywords) per language in a cell
for language in dataset_languages:
gloss_senses_of_language = sense_translations_for_language(gloss, language)
row.append(gloss_senses_of_language)
# Put example sentences per language in a cell
for language in dataset_languages:
gloss_example_sentences_of_language = sense_examplesentences_for_language(gloss, language)
row.append(gloss_example_sentences_of_language)
for f in fields:
#Try the value of the choicelist
if hasattr(f, 'field_choice_category'):
if hasattr(gloss, 'get_' + f.name + '_display'):
value = getattr(gloss, 'get_' + f.name + '_display')()
else:
field_value = getattr(gloss, f.name)
value = field_value.name if field_value else '-'
elif isinstance(f, models.ForeignKey) and f.related_model == Handshape:
handshape_field_value = getattr(gloss, f.name)
value = handshape_field_value.name if handshape_field_value else '-'
elif f.related_model == SemanticField:
value = ", ".join([str(sf.name) for sf in gloss.semField.all()])
elif f.related_model == DerivationHistory:
value = ", ".join([str(sf.name) for sf in gloss.derivHist.all()])
else:
value = getattr(gloss, f.name)
# some legacy glosses have empty text fields of other formats
if (f.__class__.__name__ == 'CharField' or f.__class__.__name__ == 'TextField') \
and value in ['-','------',' ']:
value = ''
if value is None:
if f.name in settings.HANDEDNESS_ARTICULATION_FIELDS:
value = 'Neutral'
elif f.name in settings.HANDSHAPE_ETYMOLOGY_FIELDS:
value = 'False'
else:
if hasattr(f, 'field_choice_category'):
value = '-'
elif f.__class__.__name__ == 'CharField' or f.__class__.__name__ == 'TextField':
value = ''
elif f.__class__.__name__ == 'IntegerField':
value = 0
else:
# what to do here? leave it as None or use empty string (for export to csv)
value = ''
# This was disabled with the move to Python 3... might not be needed anymore?
# if isinstance(value,unicode):
# value = str(value.encode('ascii','xmlcharrefreplace'))
if not isinstance(value,str):
# this is needed for csv
value = str(value)
# A handshape name can begin with =. To avoid Office thinking this is a formula, preface with '
# if value[:1] == '=':
# value = '\'' + value
row.append(value)
# get languages
signlanguages = [signlanguage.name for signlanguage in gloss.signlanguage.all()]
row.append(", ".join(signlanguages))
# get dialects
dialects = [dialect.name for dialect in gloss.dialect.all()]
row.append(", ".join(dialects))
# get morphology
# Sequential Morphology
morphemes = [morpheme.get_role()+':'+str(morpheme.morpheme.id) for morpheme in MorphologyDefinition.objects.filter(parent_gloss=gloss)]
row.append(", ".join(morphemes))
# Simultaneous Morphology
morphemes = [(str(m.morpheme.id), m.role) for m in gloss.simultaneous_morphology.all()]
sim_morphs = []
for m in morphemes:
sim_morphs.append(':'.join(m))
simultaneous_morphemes = ', '.join(sim_morphs)
row.append(simultaneous_morphemes)
# Blend Morphology
ble_morphemes = [(str(m.glosses.id), m.role) for m in gloss.blend_morphology.all()]
ble_morphs = []
for m in ble_morphemes:
ble_morphs.append(':'.join(m))
blend_morphemes = ', '.join(ble_morphs)
row.append(blend_morphemes)
# get relations to other signs
relations = [(relation.role, str(relation.target.id)) for relation in Relation.objects.filter(source=gloss)]
relations_with_categories = []
for rel_cat in relations:
relations_with_categories.append(':'.join(rel_cat))
relations_categories = ", ".join(relations_with_categories)
row.append(relations_categories)
# get relations to foreign signs
relations = [(str(relation.loan), relation.other_lang, relation.other_lang_gloss) for relation in RelationToForeignSign.objects.filter(gloss=gloss)]
relations_with_categories = []
for rel_cat in relations:
relations_with_categories.append(':'.join(rel_cat))
relations_categories = ", ".join(relations_with_categories)
row.append(relations_categories)
# export tags
tags_of_gloss = TaggedItem.objects.filter(object_id=gloss.id)
tag_names_of_gloss = []
for t_obj in tags_of_gloss:
tag_id = t_obj.tag_id
tag_name = Tag.objects.get(id=tag_id)
tag_names_of_gloss += [str(tag_name).replace('_',' ')]
tag_names = ", ".join(tag_names_of_gloss)
row.append(tag_names)
# export notes
notes_of_gloss = gloss.definition_set.all()
notes_list = []
for note in notes_of_gloss:
notes_list += [note.note_tuple()]
sorted_notes_list = sorted(notes_list, key=lambda x: (x[0], x[1], x[2], x[3]))
notes_list = []
for (role, published, count, text) in sorted_notes_list:
# does not use a comprehension because of nested parentheses in role and text fields
tuple_reordered = role + ': (' + published + ',' + count + ',' + text + ')'
notes_list.append(tuple_reordered)
notes_display = ", ".join(notes_list)
row.append(notes_display)
#Make it safe for weird chars
safe_row = []
for column in row:
try:
safe_row.append(column.encode('utf-8').decode())
except AttributeError:
safe_row.append(None)
writer.writerow(safe_row)
return response
def get_queryset(self):
get = self.request.GET
#First check whether we want to show everything or a subset
if 'show_all' in self.kwargs.keys():
show_all = self.kwargs['show_all']
else:
show_all = False
#Then check what kind of stuff we want
if 'search_type' in get:
self.search_type = get['search_type']
else:
self.search_type = 'sign'
setattr(self.request.session, 'search_type', self.search_type)
if 'view_type' in get:
self.view_type = get['view_type']
# don't change query, just change display
else:
# set to default
self.view_type = 'gloss_list'
setattr(self.request, 'view_type', self.view_type)
if 'inWeb' in self.request.GET:
# user is searching for signs / morphemes visible to anonymous uers
self.web_search = self.request.GET['inWeb'] == '2'
elif not self.request.user.is_authenticated:
self.web_search = True
setattr(self.request, 'web_search', self.web_search)
if self.show_all:
self.query_parameters = dict()
# erase the previous query
self.request.session['query_parameters'] = json.dumps(self.query_parameters)
self.request.session.modified = True
elif 'query_parameters' in self.request.session.keys() \
and self.request.session['query_parameters'] not in ['', '{}'] \
and 'query' in self.request.GET:
session_query_parameters = self.request.session['query_parameters']
self.query_parameters = json.loads(session_query_parameters)
if 'search_type' in self.query_parameters.keys() and self.query_parameters['search_type'] != 'sign':
# Make sure on loading the query parameters that the right kind of search is done
# this is important if the user searched on Sign or Morpheme
self.search_type = self.query_parameters['search_type']
else:
self.query_parameters = dict()
if self.request.user.is_authenticated:
selected_datasets = get_selected_datasets_for_user(self.request.user)
elif 'selected_datasets' in self.request.session.keys():
selected_datasets = Dataset.objects.filter(acronym__in=self.request.session['selected_datasets'])
else:
selected_datasets = Dataset.objects.filter(acronym=settings.DEFAULT_DATASET_ACRONYM)
dataset_languages = get_dataset_languages(selected_datasets)
from signbank.dictionary.forms import check_language_fields
valid_regex, search_fields = check_language_fields(GlossSearchForm, get, dataset_languages)
if not valid_regex:
error_message_1 = _('Error in search field ')
error_message_2 = ', '.join(search_fields)
error_message_3 = _(': Please use a backslash before special characters.')
error_message = error_message_1 + error_message_2 + error_message_3
messages.add_message(self.request, messages.ERROR, error_message)
qs = Gloss.objects.none()
return qs
#Get the initial selection
if show_all or (len(get) > 0 and 'query' not in self.request.GET):
# anonymous users can search signs, make sure no morphemes are in the results
if self.search_type == 'sign' or not self.request.user.is_authenticated:
# Get all the GLOSS items that are not member of the sub-class Morpheme
if SPEED_UP_RETRIEVING_ALL_SIGNS:
qs = Gloss.none_morpheme_objects().select_related('lemma').prefetch_related('parent_glosses').prefetch_related('simultaneous_morphology').prefetch_related('translation_set').filter(lemma__dataset__in=selected_datasets)
else:
qs = Gloss.none_morpheme_objects().filter(lemma__dataset__in=selected_datasets)
else:
if SPEED_UP_RETRIEVING_ALL_SIGNS:
qs = Gloss.objects.all().prefetch_related('lemma').prefetch_related('parent_glosses').prefetch_related('simultaneous_morphology').prefetch_related('translation_set').filter(lemma__dataset__in=selected_datasets)
else:
qs = Gloss.objects.all().filter(lemma__dataset__in=selected_datasets)
elif self.query_parameters and 'query' in self.request.GET:
if self.search_type == 'sign_or_morpheme':
qs = Gloss.objects.all().prefetch_related('lemma').filter(lemma__dataset__in=selected_datasets)
else:
qs = Gloss.none_morpheme_objects().prefetch_related('lemma').filter(lemma__dataset__in=selected_datasets)
qs = apply_language_filters_to_results(qs, self.query_parameters)
qs = qs.distinct()
query = convert_query_parameters_to_filter(self.query_parameters)
if query:
qs = qs.filter(query).distinct()
sorted_qs = order_queryset_by_sort_order(self.request.GET, qs, self.queryset_language_codes)
return sorted_qs
#No filters or 'show_all' specified? show nothing
else:
qs = Gloss.objects.none()
if self.request.user.is_authenticated and self.request.user.has_perm('dictionary.search_gloss'):
pass
else:
qs = qs.filter(inWeb__exact=True)
#If we wanted to get everything, we're done now
if show_all:
# sort the results
sorted_qs = order_queryset_by_sort_order(self.request.GET, qs, self.queryset_language_codes)
return sorted_qs
# this is a temporary query_parameters variable
# it is saved to self.query_parameters after the parameters are processed
query_parameters = dict()
#If not, we will go trhough a long list of filters
if 'search' in get and get['search'] != '':
val = get['search']
query_parameters['search'] = val
from signbank.tools import strip_control_characters
val = strip_control_characters(val)
query = Q(annotationidglosstranslation__text__iregex=val)
if re.match('^\d+$', val):
query = query | Q(sn__exact=val)
qs = qs.filter(query)
if self.search_type != 'sign':
query_parameters['search_type'] = self.search_type
# Evaluate all gloss/language search fields
for get_key, get_value in get.items():
if get_key == 'csrfmiddlewaretoken':
continue
if get_key.startswith(GlossSearchForm.gloss_search_field_prefix) and get_value != '':
query_parameters[get_key] = get_value
language_code_2char = get_key[len(GlossSearchForm.gloss_search_field_prefix):]
language = Language.objects.filter(language_code_2char=language_code_2char).first()
qs = qs.filter(annotationidglosstranslation__text__iregex=get_value,
annotationidglosstranslation__language=language)
elif get_key.startswith(GlossSearchForm.lemma_search_field_prefix) and get_value != '':
query_parameters[get_key] = get_value
language_code_2char = get_key[len(GlossSearchForm.lemma_search_field_prefix):]
language = Language.objects.filter(language_code_2char=language_code_2char).first()