From fe25f526096a5a53c43c16442ee5b7e957b9fc61 Mon Sep 17 00:00:00 2001 From: pamputt Date: Wed, 5 Apr 2023 17:20:48 +0200 Subject: [PATCH] Fix for T274511 (#22) * new script to process all recordings * list of Qidentifiers for all languages supported * Seems better (#23) * Improve script (fix some bugs) * typo --------- Co-authored-by: Lopez Hugo --- list_languages.txt | 719 +++++++++++++++++++++++++++++++++ scripts/all-languages-llbot.sh | 57 +++ 2 files changed, 776 insertions(+) create mode 100644 list_languages.txt create mode 100644 scripts/all-languages-llbot.sh diff --git a/list_languages.txt b/list_languages.txt new file mode 100644 index 0000000..e1f5287 --- /dev/null +++ b/list_languages.txt @@ -0,0 +1,719 @@ +Q21 +Q22 +Q24 +Q25 +Q26 +Q27 +Q28 +Q29 +Q30 +Q31 +Q32 +Q33 +Q34 +Q35 +Q36 +Q37 +Q38 +Q39 +Q40 +Q41 +Q43 +Q44 +Q45 +Q46 +Q47 +Q48 +Q49 +Q50 +Q51 +Q52 +Q53 +Q54 +Q55 +Q56 +Q57 +Q58 +Q59 +Q60 +Q61 +Q62 +Q63 +Q64 +Q65 +Q66 +Q67 +Q68 +Q69 +Q70 +Q71 +Q72 +Q73 +Q74 +Q75 +Q76 +Q77 +Q78 +Q79 +Q80 +Q81 +Q82 +Q83 +Q84 +Q85 +Q86 +Q87 +Q88 +Q89 +Q90 +Q91 +Q92 +Q93 +Q94 +Q95 +Q96 +Q97 +Q98 +Q99 +Q100 +Q101 +Q102 +Q103 +Q104 +Q105 +Q106 +Q107 +Q108 +Q109 +Q110 +Q111 +Q112 +Q113 +Q114 +Q115 +Q116 +Q117 +Q118 +Q119 +Q120 +Q121 +Q122 +Q123 +Q124 +Q125 +Q126 +Q127 +Q128 +Q129 +Q130 +Q131 +Q132 +Q133 +Q134 +Q135 +Q136 +Q137 +Q138 +Q139 +Q140 +Q141 +Q142 +Q143 +Q144 +Q145 +Q146 +Q147 +Q148 +Q149 +Q150 +Q151 +Q152 +Q153 +Q154 +Q155 +Q156 +Q157 +Q158 +Q159 +Q160 +Q161 +Q162 +Q163 +Q164 +Q165 +Q166 +Q167 +Q168 +Q169 +Q170 +Q171 +Q172 +Q173 +Q174 +Q175 +Q176 +Q177 +Q178 +Q179 +Q180 +Q181 +Q182 +Q183 +Q184 +Q185 +Q186 +Q187 +Q188 +Q189 +Q190 +Q191 +Q192 +Q193 +Q194 +Q195 +Q196 +Q197 +Q198 +Q199 +Q200 +Q201 +Q202 +Q203 +Q204 +Q205 +Q206 +Q207 +Q208 +Q209 +Q210 +Q211 +Q212 +Q213 +Q214 +Q215 +Q216 +Q217 +Q218 +Q219 +Q220 +Q221 +Q222 +Q223 +Q224 +Q225 +Q226 +Q227 +Q228 +Q229 +Q230 +Q231 +Q232 +Q233 +Q234 +Q235 +Q236 +Q237 +Q238 +Q239 +Q240 +Q241 +Q242 +Q243 +Q244 +Q245 +Q246 +Q247 +Q248 +Q249 +Q250 +Q251 +Q252 +Q253 +Q254 +Q255 +Q256 +Q257 +Q258 +Q259 +Q260 +Q261 +Q262 +Q263 +Q264 +Q265 +Q266 +Q267 +Q268 +Q269 +Q270 +Q271 +Q272 +Q273 +Q274 +Q275 +Q276 +Q277 +Q278 +Q279 +Q280 +Q281 +Q282 +Q283 +Q284 +Q285 +Q286 +Q287 +Q288 +Q289 +Q290 +Q291 +Q292 +Q293 +Q294 +Q295 +Q296 +Q297 +Q298 +Q299 +Q300 +Q301 +Q302 +Q303 +Q304 +Q305 +Q306 +Q307 +Q308 +Q309 +Q310 +Q311 +Q312 +Q313 +Q314 +Q315 +Q316 +Q317 +Q318 +Q319 +Q320 +Q321 +Q322 +Q323 +Q324 +Q325 +Q326 +Q327 +Q328 +Q329 +Q330 +Q331 +Q332 +Q333 +Q334 +Q335 +Q336 +Q337 +Q338 +Q339 +Q340 +Q341 +Q342 +Q343 +Q344 +Q345 +Q346 +Q347 +Q348 +Q349 +Q350 +Q351 +Q352 +Q353 +Q354 +Q355 +Q356 +Q357 +Q358 +Q359 +Q360 +Q361 +Q362 +Q363 +Q364 +Q365 +Q366 +Q367 +Q368 +Q369 +Q370 +Q371 +Q372 +Q373 +Q374 +Q375 +Q376 +Q377 +Q378 +Q379 +Q380 +Q381 +Q382 +Q383 +Q384 +Q385 +Q386 +Q387 +Q388 +Q389 +Q390 +Q391 +Q392 +Q393 +Q394 +Q395 +Q396 +Q397 +Q398 +Q399 +Q400 +Q401 +Q402 +Q403 +Q404 +Q405 +Q406 +Q407 +Q408 +Q409 +Q410 +Q411 +Q412 +Q413 +Q414 +Q415 +Q416 +Q417 +Q418 +Q419 +Q420 +Q421 +Q422 +Q423 +Q424 +Q425 +Q426 +Q427 +Q428 +Q429 +Q430 +Q431 +Q432 +Q433 +Q434 +Q435 +Q436 +Q437 +Q438 +Q439 +Q440 +Q441 +Q442 +Q443 +Q444 +Q445 +Q446 +Q447 +Q930 +Q931 +Q1185 +Q1186 +Q1187 +Q1188 +Q4465 +Q4901 +Q5049 +Q6714 +Q19858 +Q19859 +Q27302 +Q40979 +Q43042 +Q51235 +Q51236 +Q51237 +Q51299 +Q51301 +Q51302 +Q51303 +Q51304 +Q51305 +Q51306 +Q51307 +Q51308 +Q51309 +Q51310 +Q51311 +Q51312 +Q51313 +Q51314 +Q51870 +Q51871 +Q51872 +Q52067 +Q52068 +Q52069 +Q52070 +Q52072 +Q52073 +Q52074 +Q52206 +Q52207 +Q52292 +Q52293 +Q52294 +Q52295 +Q52296 +Q53356 +Q55890 +Q74902 +Q74903 +Q74904 +Q74905 +Q74906 +Q74907 +Q75180 +Q83641 +Q84029 +Q84030 +Q84031 +Q84032 +Q84033 +Q84034 +Q84035 +Q99628 +Q100311 +Q115106 +Q115107 +Q115108 +Q115109 +Q115110 +Q115111 +Q115112 +Q115113 +Q123270 +Q123271 +Q191499 +Q204940 +Q204941 +Q204942 +Q204943 +Q204946 +Q204947 +Q204948 +Q204949 +Q211851 +Q215053 +Q221062 +Q264201 +Q322719 +Q338540 +Q339099 +Q365515 +Q379019 +Q385285 +Q386221 +Q386440 +Q386441 +Q386442 +Q386443 +Q386444 +Q390278 +Q390314 +Q395932 +Q397198 +Q402272 +Q424673 +Q430329 +Q459005 +Q489393 +Q502754 +Q518078 +Q518079 +Q518087 +Q518106 +Q518118 +Q518128 +Q521069 +Q521070 +Q531093 +Q536601 +Q536602 +Q536634 +Q538619 +Q538620 +Q538621 +Q538622 +Q538623 +Q555966 +Q563459 +Q567656 +Q570501 +Q570502 +Q570503 +Q570504 +Q570505 +Q570506 +Q570507 +Q570511 +Q570512 +Q570513 +Q570514 +Q570515 +Q570516 +Q570517 +Q570519 +Q570520 +Q570521 +Q570522 +Q570523 +Q570524 +Q570525 +Q570526 +Q570527 +Q570528 +Q570529 +Q570531 +Q570532 +Q570533 +Q570534 +Q585002 +Q590184 +Q590185 +Q590186 +Q590187 +Q590188 +Q590189 +Q590190 +Q590191 +Q590192 +Q590193 +Q590194 +Q590195 +Q590196 +Q590197 +Q590198 +Q590199 +Q590200 +Q590201 +Q590202 +Q590203 +Q590204 +Q590205 +Q590206 +Q590207 +Q590208 +Q590209 +Q590210 +Q590211 +Q590212 +Q590213 +Q590214 +Q590215 +Q590216 +Q590217 +Q590218 +Q590219 +Q590220 +Q590221 +Q590222 +Q590223 +Q590225 +Q590226 +Q590227 +Q590228 +Q590229 +Q590230 +Q590231 +Q590232 +Q590233 +Q590234 +Q590235 +Q590236 +Q590237 +Q590238 +Q590239 +Q590240 +Q590241 +Q590242 +Q590243 +Q590244 +Q590245 +Q590246 +Q590247 +Q590248 +Q590249 +Q590250 +Q590251 +Q590252 +Q590253 +Q590254 +Q590255 +Q590256 +Q590257 +Q590258 +Q590259 +Q590260 +Q594447 +Q594448 +Q594449 +Q594450 +Q594451 +Q594452 +Q622446 +Q646152 +Q646153 +Q646154 +Q646155 +Q646156 +Q646157 +Q646158 +Q646159 +Q646160 +Q646162 +Q646163 +Q646164 +Q646165 +Q646166 +Q646167 +Q646168 +Q646170 +Q646175 +Q646176 +Q646177 +Q646178 +Q646179 +Q646180 +Q646181 +Q646182 +Q655433 +Q655434 +Q656403 +Q656404 +Q683870 +Q700683 +Q701734 +Q710470 +Q711231 +Q713359 +Q720542 +Q733156 +Q733159 +Q757855 +Q763811 +Q769475 +Q778085 +Q778750 +Q787855 +Q806575 +Q806576 +Q807153 diff --git a/scripts/all-languages-llbot.sh b/scripts/all-languages-llbot.sh new file mode 100644 index 0000000..69be5ce --- /dev/null +++ b/scripts/all-languages-llbot.sh @@ -0,0 +1,57 @@ +#!/bin/bash +export PYTHONUNBUFFERED=1 +# Given a list of languages supported in Lingua Libre (list_languages.txt), +# and a wiki-specific script name (${wiki_project}), +# adds missing recording to that targeted wiki. + +# Useful when a new wiki project is supported by +# Lingua Libre Bot in order to add all audio that have +# been recorded in the past + +# Define target project +wiki_project="kuwiktionary" + +# Loop on languages supported by lingualibre +while read langcode; do +# if [ "${langcode}" != "Q21" ]; then +# continue +# fi + + echo "Processing ${langcode}" + if [ "${langcode}" = "Q21" ] || # French + [ "${langcode}" = "Q298" ] || # Polish + [ "${langcode}" = "Q307" ]; # Bengali + then + # French language has too many recordings + # so splitting the request is required + # we ask for recordings for every year + current_year=`date +%Y` + for year in `seq 2017 ${current_year}`; do + for month in 01 07; do + startdate="${year}-${month}-01T00:00:00.000+00:00" + + month_end="01" + if [ ${month} = "01" ]; then + month_end="07" + fi + year_end=${year} + if [ ${month_end} = "01" ]; then + year_end=$((${year}+1)) + fi + + enddate="${year_end}-${month_end}-01T00:00:00.000+00:00" + echo "start: ${startdate}, end: ${enddate}" + $HOME/venv/bin/python3 -u $HOME/Lingua-Libre-Bot/llbot.py --wiki ${wiki_project} simple --lang ${langcode} --startdate ${startdate} --enddate ${enddate} + # sleep 5 seconds to avoid Error 429 Too Many Requests + sleep 5 + done + done + else + $HOME/venv/bin/python3 -u $HOME/Lingua-Libre-Bot/llbot.py --wiki ${wiki_project} simple --lang ${langcode} + fi + + # sleep 5 seconds to avoid Error 429 Too Many Requests + sleep 5 +done