This repository has been archived by the owner on Apr 5, 2019. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
/
cosmetic_changes.py
826 lines (757 loc) · 37 KB
/
cosmetic_changes.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
# -*- coding: utf-8 -*-
"""
This module can do slight modifications to a wiki page source code such that
the code looks cleaner. The changes are not supposed to change the look of the
rendered wiki page.
The following parameters are supported:
¶ms;
-always Don't prompt you for each replacement. Warning (see below)
has not to be confirmed. ATTENTION: Use this with care!
-summary:XYZ Set the summary message text for the edit to XYZ, bypassing
the predefined message texts with original and replacements
inserted.
All other parameters will be regarded as part of the title of a single page,
and the bot will only work on that single page.
&warning;
For regular use, it is recommended to put this line into your user-config.py:
cosmetic_changes = True
You may enable cosmetic changes for additional languages by adding the
dictionary cosmetic_changes_enable to your user-config.py. It should contain
a tuple of languages for each site where you wish to enable in addition to
your own langlanguage if cosmetic_changes_mylang_only is True (see below).
Please set your dictionary by adding such lines to your user-config.py:
cosmetic_changes_enable['wikipedia'] = ('de', 'en', 'fr')
There is another config variable: You can set
cosmetic_changes_mylang_only = False
if you're running a bot on multiple sites and want to do cosmetic changes on
all of them, but be careful if you do.
You may disable cosmetic changes by adding the all unwanted languages to the
dictionary cosmetic_changes_disable in your user-config.py. It should contain
a tuple of languages for each site where you wish to disable cosmetic changes.
You may use it with cosmetic_changes_mylang_only is False, but you can also
disable your own language. This also overrides the settings in the dictionary
cosmetic_changes_enable. Please set this dictionary by adding such lines to
your user-config.py:
cosmetic_changes_disable['wikipedia'] = ('de', 'en', 'fr')
"""
__version__ = '$Id$'
import wikipedia as pywikibot
import pagegenerators, isbn
import sys
import re
warning = """ATTENTION: You can run this script as a stand-alone for testing purposes.
However, the changes are that are made are only minor, and other users
might get angry if you fill the version histories and watchlists with such
irrelevant changes."""
docuReplacements = {
'¶ms;': pagegenerators.parameterHelp,
'&warning;': warning,
}
# Summary message when using this module as a stand-alone script
msg_standalone = {
'als':u'Bötli: chleineri Änderige',
'ar': u'روبوت: تغييرات تجميلية',
'be-x-old': u'Робат: касмэтычныя зьмены',
'bg': u'Робот козметични промени',
'br': u'Bot: Kemm dister',
'ca': u'Robot: Canvis cosmètics',
'ckb':u'بۆت: دەستکاریی جوانکاری',
'cs': u'Robotické: kosmetické úpravy',
'da': u'Bot: Kosmetiske ændringer',
'de': u'Bot: Kosmetische Änderungen',
'el': u'Ρομπότ: διακοσμητικές αλλαγές',
'en': u'Robot: Cosmetic changes',
'es': u'Robot: Cambios triviales',
'et': u'robot: kosmeetilised muudatused',
'fa': u'ربات: ویرایش جزئی',
'fi': u'Botti kosmeettisia muutoksia',
'fr': u'Robot : Changement de type cosmétique',
'fy': u'bot tekstwiziging',
'gl': u'bot Cambios estética',
'he': u'בוט: שינויים קוסמטיים',
'hi': u'Bot: अंगराग परिवर्तन',
'hr': u'robot kozmetičke promjene',
'hu': u'Bot: kozmetikai változtatások',
'ia': u'Robot: Cambios cosmetic',
'id': u'bot kosmetik perubahan',
'it': u'Bot: Modifiche estetiche',
'ja': u'ロボットによる: 細部の編集',
'ko': u'로봇: 예쁘게 바꿈',
'la': u'automaton: mutationes minores',
'lt': u'robotas: smulkūs taisymai',
'lv': u'robots kosmētiskās izmaiņas',
'mk': u'Бот: козметички промени',
'ms': u'Bot: perubahan kosmetik',
'mt': u'Bot: kosmetiċi bidliet',
'nl': u'Bot: cosmetische wijzigingen',
'no': u'Bot: Kosmetiske endringer',
'nn': u'Robot: Kosmetiske endringar',
'pdc':u'Waddefresser: gleene Enneringe',
'pl': u'Robot dokonuje poprawek kosmetycznych',
'pt': u'Bot: Mudanças triviais',
'ro': u'robot modificări cosmetice',
'ru': u'робот косметические изменения',
'sk': u'robot kozmetické zmeny',
'sl': u'robot kozmetične spremembe',
'sr': u'Бот козметичке промене',
'sv': u'Bot: Kosmetiska ändringar',
'th': u'บอต ปรับแต่งให้อ่านง่าย',
'tk': u'Bot: kosmetik üýtgeşme',
'tl': u'robot Kosmetiko pagbabago',
'tr': u'Bot Kozmetik değişiklikler',
'uk': u'робот косметичні зміни',
'vec':u'Bot: Modifiche estetiche',
'vi': u'robot: Sửa cách trình bày',
'war':u'Robot: Kosmetiko nga mga pagbag-o',
'yi': u'באט: קאסמעטישע ענדערונגען',
'zh': u'機器人: 細部更改',
}
# Summary message that will be appended to the normal message when
# cosmetic changes are made on the fly
msg_append = {
'als':u'; chleineri Änderige',
'ar': u'; تغييرات تجميلية',
'be-x-old': u'; касмэтычныя зьмены',
'bg': u'; козметични промени',
'br': u'; Kemm dister',
'ca': u'; canvis cosmètics',
'ckb':u'; دەستکاریی جوانکاری',
'cs': u'; kosmetické úpravy',
'da': u'; kosmetiske ændringer',
'de': u'; kosmetische Änderungen',
'el': u'; διακοσμητικές αλλαγές',
'en': u'; cosmetic changes',
'es': u'; cambios triviales',
'et': u'; kosmeetilised muudatused',
'fa': u'; ویرایش جزئی',
'fi': u'; kosmeettisia muutoksia',
'fr': u'; changement de type cosmétique',
'fy': u'; tekstwiziging',
'gl': u'; cambios estética',
'he': u'; שינויים קוסמטיים',
'hi': u'; अंगराग परिवर्तन',
'hr': u'; kozmetičke promjene',
'hu': u'; kozmetikai változtatások',
'ia': u'; cambios cosmetic',
'id': u'; kosmetik perubahan',
'it': u'; modifiche estetiche',
'ja': u'; 細部の編集',
'ko': u'; 예쁘게 바꿈',
'la': u'; mutationes minores',
'lt': u'; smulkūs taisymai',
'lv': u'; kosmētiskās izmaiņas',
'mt': u'; kosmetiċi bidliet',
'mk': u'; козметички промени',
'ms': u'; perubahan kosmetik',
'nl': u'; cosmetische veranderingen',
'no': u'; kosmetiske endringer',
'nn': u'; kosmetiske endringar',
'pdc':u', gleene Enneringe',
'pl': u'; zmiany kosmetyczne',
'pt': u'; mudanças triviais',
'ro': u'; modificări cosmetice',
'ru': u'; косметические изменения',
'sk': u'; kozmetické zmeny',
'sl': u'; kozmetične spremembe',
'sr': u'; козметичке промене',
'sv': u'; kosmetiska ändringar',
'th': u'; ปรับแต่งให้อ่านง่าย',
'tk': u'; kosmetik üýtgeşme',
'tl': u'; Kosmetiko pagbabago',
'tr': u'; Kozmetik değişiklikler',
'uk': u'; косметичні зміни',
'vec':u'; modifiche estetiche',
'vi': u'; sửa cách trình bày',
'war':u'; kosmetiko nga mga pagbag-o',
'yi': u'; קאסמעטישע ענדערונגען',
'zh': u'; 細部更改',
}
nn_iw_msg = u'<!--interwiki (no, sv, da first; then other languages alphabetically by name)-->'
# This is from interwiki.py;
# move it to family file and implement global instances
moved_links = {
'ca' : (u'ús de la plantilla', u'/ús'),
'cs' : (u'dokumentace', u'/doc'),
'de' : (u'dokumentation', u'/Meta'),
'en' : ([u'documentation',
u'template documentation',
u'template doc',
u'doc',
u'documentation, template'], u'/doc'),
'es' : ([u'documentación', u'documentación de plantilla'], u'/doc'),
'fr' : (u'/documentation', u'/Documentation'),
'hu' : (u'sablondokumentáció', u'/doc'),
'id' : (u'template doc', u'/doc'),
'ja' : (u'documentation', u'/doc'),
'ka' : (u'თარგის ინფო', u'/ინფო'),
'ko' : (u'documentation', u'/설명문서'),
'ms' : (u'documentation', u'/doc'),
'pl' : (u'dokumentacja', u'/opis'),
'pt' : ([u'documentação', u'/doc'], u'/doc'),
'ro' : (u'documentaţie', u'/doc'),
'ru' : (u'doc', u'/doc'),
'sv' : (u'dokumentation', u'/dok'),
'vi' : (u'documentation', u'/doc'),
'zh' : ([u'documentation', u'doc'], u'/doc'),
}
# Template which should be replaced or removed.
# Use a list with two entries. The first entry will be replaced by the second.
# Examples:
# For removing {{Foo}}, the list must be:
# (u'Foo', None),
#
# The following also works:
# (u'Foo', ''),
#
# For replacing {{Foo}} with {{Bar}} the list must be:
# (u'Foo', u'Bar'),
#
# This also removes all template parameters of {{Foo}}
# For replacing {{Foo}} with {{Bar}} but keep the template
# parameters in its original order, please use:
# (u'Foo', u'Bar\g<parameters>'),
deprecatedTemplates = {
'wikipedia': {
'de': [
(u'Stub', None),
(u'Belege', u'Belege fehlen\g<parameters>'),
(u'Quelle', u'Belege fehlen\g<parameters>'),
(u'Quellen', u'Belege fehlen\g<parameters>'),
(u'CommonsCat', u'Commonscat\g<parameters>'),
],
'pdc':[
(u'Schkiss', None),
],
}
}
class CosmeticChangesToolkit:
def __init__(self, site, debug=False, redirect=False, namespace=None, pageTitle=None):
self.site = site
self.debug = debug
self.redirect = redirect
self.namespace = namespace
self.template = (self.namespace == 10)
self.talkpage = self.namespace >= 0 and self.namespace % 2 == 1
self.title = pageTitle
def change(self, text):
"""
Given a wiki source code text, return the cleaned up version.
"""
oldText = text
text = self.fixSelfInterwiki(text)
text = self.standardizePageFooter(text)
text = self.cleanUpLinks(text)
text = self.cleanUpSectionHeaders(text)
text = self.putSpacesInLists(text)
text = self.translateAndCapitalizeNamespaces(text)
text = self.replaceDeprecatedTemplates(text)
text = self.resolveHtmlEntities(text)
text = self.validXhtml(text)
text = self.removeUselessSpaces(text)
text = self.removeNonBreakingSpaceBeforePercent(text)
text = self.fixSyntaxSave(text)
text = self.fixHtml(text)
text = self.fixStyle(text)
text = self.fixTypo(text)
text = self.fixArabicLetters(text)
try:
text = isbn.hyphenateIsbnNumbers(text)
except isbn.InvalidIsbnException, error:
if pywikibot.verbose:
pywikibot.output(u"ISBN error: %s" % error)
pass
if self.debug:
pywikibot.showDiff(oldText, text)
return text
def fixSelfInterwiki(self, text):
"""
Interwiki links to the site itself are displayed like local links.
Remove their language code prefix.
"""
if not self.talkpage and pywikibot.calledModuleName() <> 'interwiki':
interwikiR = re.compile(r'\[\[%s\s?:([^\[\]\n]*)\]\]' % self.site.lang)
text = interwikiR.sub(r'[[\1]]', text)
return text
def standardizePageFooter(self, text):
"""
Makes sure that interwiki links, categories and star templates are
put to the correct position and into the right order.
This combines the old instances standardizeInterwiki and standardizeCategories
The page footer has the following section in that sequence:
1. categories
2. additional information depending on local site policy
3. stars templates for featured and good articles
4. interwiki links
"""
starsList = [
u'bueno',
u'cyswllt[ _]erthygl[ _]ddethol', u'dolen[ _]ed',
u'destacado', u'destaca[tu]',
u'enllaç[ _]ad',
u'enllaz[ _]ad',
u'leam[ _]vdc',
u'legătură[ _]a[bcf]',
u'liamm[ _]pub',
u'lien[ _]adq',
u'lien[ _]ba',
u'liên[ _]kết[ _]bài[ _]chất[ _]lượng[ _]tốt',
u'liên[ _]kết[ _]chọn[ _]lọc',
u'ligam[ _]adq',
u'ligoelstara',
u'ligoleginda',
u'link[ _][afgu]a', u'link[ _]adq', u'link[ _]f[lm]', u'link[ _]km', u'link[ _]sm', u'linkfa',
u'na[ _]lotura',
u'nasc[ _]ar',
u'tengill[ _][úg]g',
u'ua',
u'yüm yg',
u'רא',
u'وصلة مقالة جيدة',
u'وصلة مقالة مختارة',
]
categories = None
interwikiLinks = None
allstars = []
hasCommentLine = False
# The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia.
# See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/bis_2006#Position_der_Personendaten_am_.22Artikelende.22
# ignoring nn-wiki of cause of the comment line above iw section
if not self.template and not '{{Personendaten' in text:
categories = pywikibot.getCategoryLinks(text, site = self.site)
if not self.talkpage:# and pywikibot.calledModuleName() <> 'interwiki':
subpage = False
if self.template:
loc = None
try:
tmpl, loc = moved_links[self.site.lang]
del tmpl
except KeyError:
pass
if loc != None and loc in self.title:
subpage = True
interwikiLinks = pywikibot.getLanguageLinks(text, insite=self.site, template_subpage=subpage)
# Removing the interwiki
text = pywikibot.removeLanguageLinks(text, site = self.site)
# Removing the stars' issue
starstext = pywikibot.removeDisabledParts(text)
for star in starsList:
regex = re.compile('(\{\{(?:template:|)%s\|.*?\}\}[\s]*)' % star, re.I)
found = regex.findall(starstext)
if found != []:
if pywikibot.verbose:
print found
text = regex.sub('', text)
allstars += found
# nn got a message between the categories and the iw's
# and they want to keep it there, first remove it
if self.site.language()=='nn':
regex = re.compile('(<!-- ?interwiki \(no(/nb)?, ?sv, ?da first; then other languages alphabetically by name\) ?-->)')
found = regex.findall(text)
if found:
if pywikibot.verbose:
print found
hasCommentLine = True
text = regex.sub('', text)
# Adding categories
if categories:
text = pywikibot.replaceCategoryLinks(text, categories, site = self.site)
# Put the nn iw message back
if self.site.language()=='nn' and not self.talkpage and (interwikiLinks or hasCommentLine):
text = text + '\r\n\r\n' + nn_iw_msg
# Adding stars templates
if allstars:
text = text.strip()+self.site.family.interwiki_text_separator
allstars.sort()
for element in allstars:
text += '%s\r\n' % element.strip()
if pywikibot.verbose:
pywikibot.output(u'%s' %element.strip())
# Adding the interwiki
if interwikiLinks:
text = pywikibot.replaceLanguageLinks(text, interwikiLinks, site = self.site, template = self.template, template_subpage = subpage)
return text
def translateAndCapitalizeNamespaces(self, text):
"""
Makes sure that localized namespace names are used.
"""
# arz uses english stylish codes
if self.site.sitename() == 'wikipedia:arz':
return text
family = self.site.family
# wiki links aren't parsed here.
exceptions = ['nowiki', 'comment', 'math', 'pre']
for nsNumber in family.namespaces:
if not family.isDefinedNSLanguage(nsNumber, self.site.lang):
# Skip undefined namespaces
continue
namespaces = list(self.site.namespace(nsNumber, all = True))
thisNs = namespaces.pop(0)
if nsNumber == 6 and family.name == 'wikipedia' and \
self.site.lang in ('en', 'fr'):
# do not change "Image" on en-wiki and fr-wiki
for image in [u'Image', u'image']:
if image in namespaces:
namespaces.remove(image)
# skip main (article) namespace
if thisNs and namespaces:
text = pywikibot.replaceExcept(text, r'\[\[\s*(' + '|'.join(namespaces) + ') *:(?P<nameAndLabel>.*?)\]\]', r'[[' + thisNs + ':\g<nameAndLabel>]]', exceptions)
return text
def cleanUpLinks(self, text):
# helper function which works on one link and either returns it
# unmodified, or returns a replacement.
def handleOneLink(match):
titleWithSection = match.group('titleWithSection')
label = match.group('label')
trailingChars = match.group('linktrail')
newline = match.group('newline')
if not self.site.isInterwikiLink(titleWithSection):
# The link looks like this:
# [[page_title|link_text]]trailing_chars
# We only work on namespace 0 because pipes and linktrails work
# differently for images and categories.
try:
page = pywikibot.Page(self.site, titleWithSection)
#except pywikibot.InvalidTitle:
except: #empty self link occures
return match.group()
if page.namespace() == 0:
# Replace underlines by spaces, also multiple underlines
titleWithSection = re.sub('_+', ' ', titleWithSection)
# Remove double spaces
titleWithSection = re.sub(' +', ' ', titleWithSection)
# Remove unnecessary leading spaces from title,
# but remember if we did this because we eventually want
# to re-add it outside of the link later.
titleLength = len(titleWithSection)
titleWithSection = titleWithSection.lstrip()
hadLeadingSpaces = (len(titleWithSection) != titleLength)
hadTrailingSpaces = False
# Remove unnecessary trailing spaces from title,
# but remember if we did this because it may affect
# the linktrail and because we eventually want to
# re-add it outside of the link later.
if not trailingChars:
titleLength = len(titleWithSection)
titleWithSection = titleWithSection.rstrip()
hadTrailingSpaces = (len(titleWithSection) != titleLength)
# Convert URL-encoded characters to unicode
titleWithSection = pywikibot.url2unicode(titleWithSection, site = self.site)
if titleWithSection == '':
# just skip empty links.
return match.group()
# Remove unnecessary initial and final spaces from label.
# Please note that some editors prefer spaces around pipes. (See [[en:Wikipedia:Semi-bots]]). We remove them anyway.
if label is not None:
# Remove unnecessary leading spaces from label,
# but remember if we did this because we want
# to re-add it outside of the link later.
labelLength = len(label)
label = label.lstrip()
hadLeadingSpaces = (len(label) != labelLength)
# Remove unnecessary trailing spaces from label,
# but remember if we did this because it affects
# the linktrail.
if not trailingChars:
labelLength = len(label)
label = label.rstrip()
hadTrailingSpaces = (len(label) != labelLength)
else:
label = titleWithSection
if trailingChars:
label += trailingChars
if titleWithSection == label or titleWithSection[0].lower() + titleWithSection[1:] == label:
newLink = "[[%s]]" % label
# Check if we can create a link with trailing characters instead of a pipelink
elif len(titleWithSection) <= len(label) and label[:len(titleWithSection)] == titleWithSection and re.sub(trailR, '', label[len(titleWithSection):]) == '':
newLink = "[[%s]]%s" % (label[:len(titleWithSection)], label[len(titleWithSection):])
else:
# Try to capitalize the first letter of the title.
# Maybe this feature is not useful for languages that
# don't capitalize nouns...
#if not self.site.nocapitalize:
if self.site.sitename() == 'wikipedia:de':
titleWithSection = titleWithSection[0].upper() + titleWithSection[1:]
newLink = "[[%s|%s]]" % (titleWithSection, label)
# re-add spaces that were pulled out of the link.
# Examples:
# text[[ title ]]text -> text [[title]] text
# text[[ title | name ]]text -> text [[title|name]] text
# text[[ title |name]]text -> text[[title|name]]text
# text[[title| name]]text -> text [[title|name]]text
if hadLeadingSpaces and not newline:
newLink = ' ' + newLink
if hadTrailingSpaces:
newLink = newLink + ' '
if newline:
newLink = newline + newLink
return newLink
# don't change anything
return match.group()
trailR = re.compile(self.site.linktrail())
# The regular expression which finds links. Results consist of four groups:
# group title is the target page title, that is, everything before | or ].
# group section is the page section. It'll include the # to make life easier for us.
# group label is the alternative link title, that's everything between | and ].
# group linktrail is the link trail, that's letters after ]] which are part of the word.
# note that the definition of 'letter' varies from language to language.
linkR = re.compile(r'(?P<newline>[\n]*)\[\[(?P<titleWithSection>[^\]\|]+)(\|(?P<label>[^\]\|]*))?\]\](?P<linktrail>' + self.site.linktrail() + ')')
text = pywikibot.replaceExcept(text, linkR, handleOneLink, ['comment', 'math', 'nowiki', 'pre', 'startspace'])
return text
def resolveHtmlEntities(self, text):
ignore = [
38, # Ampersand (&)
60, # Less than (<)
62, # Great than (>)
91, # Opening bracket - sometimes used intentionally inside links
93, # Closing bracket - sometimes used intentionally inside links
124, # Vertical bar (??) - used intentionally in navigation bar templates on de:
160, # Non-breaking space ( ) - not supported by Firefox textareas
]
# ignore ' see http://eo.wikipedia.org/w/index.php?title=Liberec&diff=next&oldid=2320801
if self.site.lang == 'eo':
ignore += [39]
text = pywikibot.html2unicode(text, ignore = ignore)
return text
def validXhtml(self, text):
text = pywikibot.replaceExcept(text, r'(?i)<br[ /]*>', r'<br />', ['comment', 'math', 'nowiki', 'pre'])
return text
def removeUselessSpaces(self, text):
result = []
multipleSpacesR = re.compile(' +')
spaceAtLineEndR = re.compile(' $')
exceptions = ['comment', 'math', 'nowiki', 'pre', 'startspace', 'table', 'template']
text = pywikibot.replaceExcept(text, multipleSpacesR, ' ', exceptions)
text = pywikibot.replaceExcept(text, spaceAtLineEndR, '', exceptions)
return text
def removeNonBreakingSpaceBeforePercent(self, text):
'''
Newer MediaWiki versions automatically place a non-breaking space in
front of a percent sign, so it is no longer required to place it
manually.
'''
text = pywikibot.replaceExcept(text, r'(\d) %', r'\1 %', ['timeline'])
return text
def cleanUpSectionHeaders(self, text):
"""
For better readability of section header source code, puts a space
between the equal signs and the title.
Example: ==Section title== becomes == Section title ==
NOTE: This space is recommended in the syntax help on the English and
German Wikipedia. It might be that it is not wanted on other wikis.
If there are any complaints, please file a bug report.
"""
for level in range(1, 7):
equals = '=' * level
text = pywikibot.replaceExcept(text, r'\n' + equals + ' *(?P<title>[^=]+?) *' + equals + ' *\r\n', '\n' + equals + ' \g<title> ' + equals + '\r\n', ['comment', 'math', 'nowiki', 'pre'])
return text
def putSpacesInLists(self, text):
"""
For better readability of bullet list and enumeration wiki source code,
puts a space between the * or # and the text.
NOTE: This space is recommended in the syntax help on the English, German,
and French Wikipedia. It might be that it is not wanted on other wikis.
If there are any complaints, please file a bug report.
"""
exceptions = ['comment', 'math', 'nowiki', 'pre', 'source', 'timeline']
if not self.redirect and pywikibot.calledModuleName() <> 'capitalize_redirects':
text = pywikibot.replaceExcept(text, r'(?m)^(?P<bullet>[:;]*(\*+|#+)[:;\*#]*)(?P<char>[^\s\*#:;].+?)', '\g<bullet> \g<char>', exceptions)
return text
def replaceDeprecatedTemplates(self, text):
exceptions = ['comment', 'math', 'nowiki', 'pre']
if self.site.family.name in deprecatedTemplates and self.site.lang in deprecatedTemplates[self.site.family.name]:
for template in deprecatedTemplates[self.site.family.name][self.site.lang]:
old = template[0]
new = template[1]
if new == None:
new = ''
else:
new = '{{'+new+'}}'
if not self.site.nocapitalize:
old = '[' + old[0].upper() + old[0].lower() + ']' + old[1:]
text = pywikibot.replaceExcept(text, r'\{\{([mM][sS][gG]:)?' + old + '(?P<parameters>\|[^}]+|)}}', new, exceptions)
return text
#from fixes.py
def fixSyntaxSave(self, text):
exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace']
# external link in double brackets
text = pywikibot.replaceExcept(text, r'\[\[(?P<url>https?://[^\]]+?)\]\]', r'[\g<url>]', exceptions)
# external link starting with double bracket
text = pywikibot.replaceExcept(text, r'\[\[(?P<url>https?://.+?)\]', r'[\g<url>]', exceptions)
# external link and description separated by a dash, with
# whitespace in front of the dash, so that it is clear that
# the dash is not a legitimate part of the URL.
text = pywikibot.replaceExcept(text, r'\[(?P<url>https?://[^\|\] \r\n]+?) +\| *(?P<label>[^\|\]]+?)\]', r'[\g<url> \g<label>]', exceptions)
# dash in external link, where the correct end of the URL can
# be detected from the file extension. It is very unlikely that
# this will cause mistakes.
text = pywikibot.replaceExcept(text, r'\[(?P<url>https?://[^\|\] ]+?(\.pdf|\.html|\.htm|\.php|\.asp|\.aspx|\.jsp)) *\| *(?P<label>[^\|\]]+?)\]', r'[\g<url> \g<label>]', exceptions)
return text
def fixHtml(self, text):
# Everything case-insensitive (?i)
# Keep in mind that MediaWiki automatically converts <br> to <br />
exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace']
text = pywikibot.replaceExcept(text, r'(?i)<b>(.*?)</b>', r"'''\1'''" , exceptions)
text = pywikibot.replaceExcept(text, r'(?i)<strong>(.*?)</strong>', r"'''\1'''" , exceptions)
text = pywikibot.replaceExcept(text, r'(?i)<i>(.*?)</i>', r"''\1''" , exceptions)
text = pywikibot.replaceExcept(text, r'(?i)<em>(.*?)</em>', r"''\1''" , exceptions)
# horizontal line without attributes in a single line
text = pywikibot.replaceExcept(text, r'(?i)([\r\n])<hr[ /]*>([\r\n])', r'\1----\2', exceptions)
# horizontal line with attributes; can't be done with wiki syntax
# so we only make it XHTML compliant
text = pywikibot.replaceExcept(text, r'(?i)<hr ([^>/]+?)>', r'<hr \1 />', exceptions)
# a header where only spaces are in the same line
for level in range(1, 7):
equals = '\\1%s \\2 %s\\3' % ("="*level, "="*level)
text = pywikibot.replaceExcept(text,
r'(?i)([\r\n]) *<h%d> *([^<]+?) *</h%d> *([\r\n])'%(level, level),
r'%s'%equals, exceptions)
#remove empty <ref/>-tag
text = pywikibot.replaceExcept(text, r'(?i)<ref\s*/>', r'', exceptions)
# TODO: maybe we can make the bot replace <p> tags with \r\n's.
return text
def fixStyle(self, text):
exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace']
# convert prettytable to wikitable class
if self.site.language in ('de', 'en'):
text = pywikibot.replaceExcept(text, ur'(class="[^"]*)prettytable([^"]*")', ur'\1wikitable\2', exceptions)
return text
def fixTypo(self, text):
exceptions = ['nowiki', 'comment', 'math', 'pre', 'source', 'startspace', 'gallery', 'hyperlink', 'interwiki', 'link']
# change <number> ccm -> <number> cm³
text = pywikibot.replaceExcept(text, ur'(\d)\s* ccm', ur'\1 cm³', exceptions)
text = pywikibot.replaceExcept(text, ur'(\d)\s*ccm', ur'\1 cm³', exceptions)
# Solve wrong Nº sign with °C or °F
# additional exception requested on fr-wiki for this stuff
pattern = re.compile(u'«.*?»', re.UNICODE)
exceptions.append(pattern)
text = pywikibot.replaceExcept(text, ur'(\d)\s* [º°]([CF])', ur'\1 °\2', exceptions)
text = pywikibot.replaceExcept(text, ur'(\d)\s*[º°]([CF])', ur'\1 °\2', exceptions)
text = pywikibot.replaceExcept(text, ur'º([CF])', ur'°\1', exceptions)
return text
def fixArabicLetters(self, text):
if self.site.lang=='ckb':
exceptions = [
'gallery',
'hyperlink',
'interwiki',
# but changes letters inside wikilinks
#'link',
'math',
'pre',
'template',
'timeline',
'ref',
'source',
'startspace',
'inputbox',
]
# do not change inside file links
namespaces = list(self.site.namespace(6, all = True))
pattern = re.compile(u'\[\[(' + '|'.join(namespaces) + '):.+?\..+?\]\]', re.UNICODE)
exceptions.append(pattern)
text = pywikibot.replaceExcept(text, u',', u'،', exceptions)
text = pywikibot.replaceExcept(text, ur'ه([.،_<\]\s])', ur'ە\1', exceptions)
text = pywikibot.replaceExcept(text, u'ه', u'ە', exceptions)
text = pywikibot.replaceExcept(text, u'ه', u'ھ', exceptions)
text = pywikibot.replaceExcept(text, u'ك', u'ک', exceptions)
text = pywikibot.replaceExcept(text, ur'[ىي]', u'ی', exceptions)
# replace persian digits
for i in range(0,10):
text = pywikibot.replaceExcept(text, u'۰۱۲۳۴۵۶۷۸۹'[i], u'٠١٢٣٤٥٦٧٨٩'[i], exceptions)
# do not change digits in class, style and table params
pattern = re.compile(u'=".*?"', re.UNICODE)
exceptions.append(pattern)
# do not change digits inside html-tags
pattern = re.compile(u'<[/]*?[^</]+?[/]*?>', re.UNICODE)
exceptions.append(pattern)
for i in range(0,10):
text = pywikibot.replaceExcept(text, str(i), u'٠١٢٣٤٥٦٧٨٩'[i], exceptions)
return text
class CosmeticChangesBot:
def __init__(self, generator, acceptall = False, comment=u'Robot: Cosmetic changes'):
self.generator = generator
self.acceptall = acceptall
self.comment = comment
self.done = False
def treat(self, page):
try:
# Show the title of the page we're working on.
# Highlight the title in purple.
pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title())
ccToolkit = CosmeticChangesToolkit(page.site(), debug = True, namespace = page.namespace(), pageTitle=page.title())
changedText = ccToolkit.change(page.get())
if changedText.strip() != page.get().strip():
if not self.acceptall:
choice = pywikibot.inputChoice(u'Do you want to accept these changes?',
['Yes', 'No', 'All', 'Quit'], ['y', 'N', 'a', 'q'], 'N')
if choice == 'a':
self.acceptall = True
elif choice == 'q':
self.done = True
return
if self.acceptall or choice == 'y':
page.put(changedText, comment=self.comment)
else:
pywikibot.output('No changes were necessary in %s' % page.title())
except pywikibot.NoPage:
pywikibot.output("Page %s does not exist?!" % page.aslink())
except pywikibot.IsRedirectPage:
pywikibot.output("Page %s is a redirect; skipping." % page.aslink())
except pywikibot.LockedPage:
pywikibot.output("Page %s is locked?!" % page.aslink())
except pywikibot.EditConflict:
pywikibot.output("An edit conflict has occured at %s." % page.aslink())
def run(self):
try:
for page in self.generator:
if self.done: break
self.treat(page)
except KeyboardInterrupt:
pywikibot.output('\nQuitting program...')
def main():
#page generator
gen = None
pageTitle = []
editSummary = ''
answer = 'y'
always = False
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory()
for arg in pywikibot.handleArgs():
if arg.startswith('-summary:'):
editSummary = arg[len('-summary:'):]
elif arg == '-always':
always = True
elif not genFactory.handleArg(arg):
pageTitle.append(arg)
if editSummary == '':
# Load default summary message.
editSummary = pywikibot.translate(pywikibot.getSite(), msg_standalone)
# Disabled this check. Although the point is still valid, there
# is now a warning and a prompt (see below).
#if pywikibot.getSite() == pywikibot.getSite('nl','wikipedia'):
#print "Deze bot is op WikipediaNL niet gewenst."
#print "Het toevoegen van cosmetic changes bij andere wijzigingen is toegestaan,"
#print "maar cosmetic_changes als stand-alone bot niet."
#print "Zoek alstublieft een nuttig gebruik voor uw bot."
#sys.exit()
if pageTitle:
page = pywikibot.Page(pywikibot.getSite(), ' '.join(pageTitle))
gen = iter([page])
if not gen:
gen = genFactory.getCombinedGenerator()
if not gen:
pywikibot.showHelp()
elif not always:
answer = pywikibot.inputChoice(warning + '\nDo you really want to continue?', ['yes', 'no'], ['y', 'N'], 'N')
if answer == 'y':
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = CosmeticChangesBot(preloadingGen, acceptall=always, comment=editSummary)
bot.run()
if __name__ == "__main__":
try:
main()
finally:
pywikibot.stopme()