-
Notifications
You must be signed in to change notification settings - Fork 0
/
init-work-ISL.sh
58 lines (40 loc) · 1.97 KB
/
init-work-ISL.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#!/bin/bash
#export H="/media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/"
#export M="/dev/shm/wikinews/"
# example article :
# /dev/shm/wikinews/articles_cleaned/86383-Al_Sharpton_speaks_out_on_race,_rights_and_w
mkdir /dev/shm/wikinews
# 2: extract the pages
cp /media/aschumilin/MYLINUXLIVE/AIFB/datasets/en/wikinews-pages.tar.gz /dev/shm/wikinews/
cd /dev/shm/wikinews/ # ohne gehts nicht
tar -zxf wikinews-pages.tar.gz
mv /dev/shm/wikinews/dev/shm/wikinews/* /dev/shm/wikinews/
# 2.1: cleanup
rm -fR /dev/shm/wikinews/dev/
rm /dev/shm/wikinews/wikinews-pages.tar.gz
# 3: make a target dir for good pages
mkdir /dev/shm/wikinews/articles_cleaned
# 4: execute filter2
python /media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/parser/filter2.py
# 5: manual cleaning of bad pages
rm /dev/shm/wikinews/articles_cleaned/*Main_Page*
rm /dev/shm/wikinews/articles_cleaned/*-Colleges_offering_admission_to_displaced_New
rm /dev/shm/wikinews/articles_cleaned/54162-2006_U.S._Congressional_Elections
rm /dev/shm/wikinews/articles_cleaned/180311-2010_UK_general_election_results
rm /dev/shm/wikinews/articles_cleaned/8636-Results_of_2005_United_Kingdom_General_Electi
rm /dev/shm/wikinews/articles_cleaned/*-Australia-200*
rm /dev/shm/wikinews/articles_cleaned/*News_briefs*
rm /dev/shm/wikinews/articles_cleaned/*Wikinews_Shorts*
rm /dev/shm/wikinews/articles_cleaned/*-Crosswords-*
# these contain broken wiki markup syntax
rm /dev/shm/wikinews/articles_cleaned/207413-*
rm /dev/shm/wikinews/articles_cleaned/562833-*
# 6: extract candidate entities from each article (entities & lang-links)
python /media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/parser/filter3.py
#7: separate lang-links from entities
python /media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/parser/filter4.py
#8: remove articles containing less than (<) X entity mentions (now, threshold is 3)
python /media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/parser/filter5.py
#9: build tensor
python /media/aschumilin/MYLINUXLIVE/AIFB/Wikinews/tensor/make.py
echo "done"