Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Initial commit

  • Loading branch information...
commit e86bfa9f50716b996641ea26a2177ebbb163fb40 0 parents
@Motiejus authored
55 README.md
@@ -0,0 +1,55 @@
+### Rynanair crawler that uses Webkit as backend ###
+
+Works as of 2011-01-08. Then I found Azuon, which
+made my work unnecessary.
+Beware, this thing does not show good coding practices. It is more a working prototype.
+
+### Usage ###
+
+## For single instance ##
+
+ryanaid.jar is a java runnable. You will need some kind of JRE.
+It takes environment variables as parameters:
+date_from="2010-03-11" # Start checking at this date
+check_days=90 # Finish at ~ may 11'th
+from="KUN" # Airport code, Kaunas in this case
+to="EDI" # Edinburgh
+wait_ms=5000 # Wait this number of ms after every request (click). Useful if you do not want to be banned from the ryanair website.
+socks_host=127.0.0.1 # Optional. Connect through this place
+socks_port=1080 # Self explanatory if you know what SOCKS is
+
+All variables have default values, so you can just launch it like this:
+
+$ from="KUN" to="HHN" java -jar ryanaid.jar
+
+You may have firebug installed. Check source for details.
+
+## For multiple firefox instances (real automatic crawling) ##
+
+Here are helpers that help to crawl all the ryanair. You will need:
+* (da|a|ba|c)sh (a _shell_)
+* autossh (for reliable socks proxies)
+* crawler_key and crawler_key.pub, which are rsa/dsa ssh key pairs (to connect to your reliable socks-via-ssh proxies)
+Basically your crawling speed is limited by how many hosts and RAM you have.
+9 crawlers were working full-time, used ~2.5 GiB RAM and ~150kBps bandwidth as average (90% - download).
+Fetched all ryanair flights (now-3 months) approximately once a day.
+
+$ ssh-keygen -t rsa <ENTER><ENTER>
+$ mv ~/.ssh/id_rsa crawler_key
+$ mv ~/.ssh/id_rsa.pub crawler_key.pub
+
+$ ./download_dests > dests.txt # Renew destination pairs
+## Fill all the hosts you can get access to to the tunnel_hosts.txt. Example:
+127.0.0.1:22
+srv.example.com:22
+wrt.example.com:2222
+## Upload your key to host so they you connect using your private key (http://www.google.com/search?q=authorized_keys)
+$ ./go_1crawler 1 &
+$ ./go_1crawler 2 &
+$ ./go_1crawler 3 &
+and so on, repeat for each host you have in your tunnel_hosts.txt.
+
+actiondir/ is logs and pids of autossh
+datadir/ is the thing you want to process.
+
+Sorry for writing style. Patches for this README are very welcome :)
2  actiondir/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
0  crawler_key
No changes.
0  crawler_key.pub
No changes.
2  datadir/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
2,378 dests.txt
@@ -0,0 +1,2378 @@
+LIG-BRS
+LIG-EMA
+LIG-LBA
+LIG-LPL
+LIG-STN
+LIL-GRO
+LIL-MRS
+LIL-OPO
+RYG-AAR
+RYG-ALC
+RYG-BCN
+RYG-SXF
+RYG-BZR
+RYG-CRL
+RYG-DUB
+RYG-NRN
+RYG-FAO
+RYG-GDN
+RYG-IBZ
+RYG-KUN
+RYG-KRK
+RYG-LRH
+RYG-LPL
+RYG-LGW
+RYG-STN
+RYG-MAD
+RYG-AGP
+RYG-FMM
+RYG-MUN
+RYG-NCL
+RYG-PMO
+RYG-PMI
+RYG-BVA
+RYG-XCR
+RYG-RIX
+RYG-CIA
+RYG-TLL
+RYG-TMP
+RYG-VLC
+RYG-TSF
+RYG-WRO
+RYG-ZAD
+ILD-HHN
+ILD-BGY
+AGA-CRL
+AGA-NRN
+AGA-HHN
+AGA-STN
+AGA-MRS
+BOH-ALC
+BOH-FAO
+BOH-FUE
+BOH-GRO
+BOH-LPA
+BOH-IBZ
+BOH-ACE
+BOH-AGP
+BOH-MJV
+BOH-PMI
+BOH-PSA
+BOH-TFS
+GSE-AHO
+GSE-ALC
+GSE-CRL
+GSE-NRN
+GSE-EDI
+GSE-HHN
+GSE-GRO
+GSE-KUN
+GSE-KLU
+GSE-STN
+GSE-AGP
+GSE-MRS
+GSE-BGY
+GSE-BVA
+GSE-PSA
+GSE-CIA
+GSE-TPS
+NYO-AHO
+NYO-ALC
+NYO-SXF
+NYO-BZR
+NYO-BIQ
+NYO-BTS
+NYO-BRE
+NYO-BDS
+NYO-CRL
+NYO-DUB
+NYO-NRN
+NYO-EDI
+NYO-EIN
+NYO-FAO
+NYO-HHN
+NYO-GRO
+NYO-GNB
+NYO-LBC
+NYO-IBZ
+NYO-FKB
+NYO-KLU
+NYO-KRK
+NYO-LPL
+NYO-LGW
+NYO-STN
+NYO-MAD
+NYO-AGP
+NYO-MLA
+NYO-MRS
+NYO-FMM
+NYO-BGY
+NYO-MUN
+NYO-PMI
+NYO-BVA
+NYO-XCR
+NYO-PSA
+NYO-RIX
+NYO-RMI
+NYO-CIA
+NYO-TLL
+NYO-TPS
+NYO-TSF
+NYO-ZAD
+EGC-BRS
+EGC-CRL
+EGC-EMA
+EGC-LPL
+EGC-STN
+BGY-AHO
+BGY-ALC
+BGY-BCN
+BGY-BRI
+BGY-SXF
+BGY-BLL
+BGY-BTS
+BGY-BRE
+BGY-BDS
+BGY-BRS
+BGY-BRQ
+BGY-CRL
+BGY-CAG
+BGY-CND
+BGY-ORK
+BGY-DUB
+BGY-NRN
+BGY-EMA
+BGY-EIN
+BGY-FEZ
+BGY-HHN
+BGY-FUE
+BGY-GRO
+BGY-PIK
+BGY-GSE
+BGY-GRX
+BGY-LPA
+BGY-LBC
+BGY-IBZ
+BGY-KUN
+BGY-KGS
+BGY-KRK
+BGY-SUF
+BGY-LPP
+BGY-LPL
+BGY-ILD
+BGY-LCJ
+BGY-LGW
+BGY-STN
+BGY-LDE
+BGY-MAD
+BGY-AGP
+BGY-RAK
+BGY-TRF
+BGY-PMO
+BGY-BVA
+BGY-PSR
+BGY-OPO
+BGY-POZ
+BGY-REU
+BGY-RHO
+BGY-RIX
+BGY-CIA
+BGY-SDR
+BGY-SCQ
+BGY-SVQ
+BGY-NYO
+BGY-TLL
+BGY-TMP
+BGY-TNG
+BGY-TFS
+BGY-TPS
+BGY-VLC
+BGY-VLL
+BGY-VOL
+BGY-WRO
+BGY-ZAZ
+CCF-CRL
+CCF-ORK
+CCF-DUB
+CCF-EMA
+CCF-PIK
+CCF-LBA
+CCF-LPL
+CCF-STN
+CCF-OPO
+BOD-BLQ
+BOD-CRL
+BOD-ORK
+BOD-EDI
+BOD-OPO
+FNI-CRL
+FNI-LPL
+FNI-LTN
+AGP-AAR
+AGP-BCN
+AGP-SXF
+AGP-BLL
+AGP-BHX
+AGP-BLQ
+AGP-BOH
+AGP-BTS
+AGP-BRE
+AGP-BRS
+AGP-CRL
+AGP-ORK
+AGP-DUB
+AGP-NRN
+AGP-EMA
+AGP-EDI
+AGP-EIN
+AGP-HHN
+AGP-PIK
+AGP-GSE
+AGP-HAU
+AGP-IBZ
+AGP-KRK
+AGP-LBA
+AGP-LPL
+AGP-STN
+AGP-MST
+AGP-CSO
+AGP-MRS
+AGP-FMM
+AGP-BGY
+AGP-MUN
+AGP-RYG
+AGP-TRF
+AGP-BVA
+AGP-PSA
+AGP-SDR
+AGP-SCQ
+AGP-SNN
+AGP-NYO
+AGP-VST
+AGP-TMP
+AGP-VLC
+AGP-VLL
+AGP-TSF
+AGP-WRO
+AGP-ZAZ
+SZZ-DUB
+SZZ-GRO
+SZZ-LPL
+SZZ-STN
+PIK-ALC
+PIK-CCF
+PIK-LDY
+PIK-DUB
+PIK-FAO
+PIK-FUE
+PIK-GRO
+PIK-LPA
+PIK-IBZ
+PIK-ACE
+PIK-STN
+PIK-AGP
+PIK-BGY
+PIK-MJV
+PIK-PMI
+PIK-BVA
+PIK-PSA
+PIK-REU
+PIK-RIX
+PIK-CIA
+PIK-SZG
+PIK-TFS
+PIK-WRO
+AOC-STN
+SXF-DUB
+SXF-EMA
+SXF-EDI
+SXF-KUN
+SXF-STN
+SXF-AGP
+SXF-BGY
+SXF-RYG
+SXF-NYO
+RAK-ALC
+RAK-BLQ
+RAK-BRE
+RAK-BRS
+RAK-CRL
+RAK-NRN
+RAK-EMA
+RAK-EDI
+RAK-HHN
+RAK-GRO
+RAK-LTN
+RAK-STN
+RAK-MAD
+RAK-MRS
+RAK-BGY
+RAK-BVA
+RAK-PSA
+RAK-OPO
+RAK-REU
+RAK-CIA
+RAK-SVQ
+RAK-VLC
+AOI-AHO
+AOI-CRL
+AOI-NRN
+AOI-STN
+AOI-MAD
+AOI-TPS
+TLN-LPL
+TLN-STN
+LBA-ALC
+LBA-BCN
+LBA-CCF
+LBA-DUB
+LBA-NRN
+LBA-FAO
+LBA-FUE
+LBA-GDN
+LBA-IBZ
+LBA-NOC
+LBA-KRK
+LBA-ACE
+LBA-LIG
+LBA-AGP
+LBA-MLA
+LBA-MPL
+LBA-MJV
+LBA-NTE
+LBA-PMI
+LBA-PSA
+LBA-TSF
+LBC-GRO
+LBC-STN
+LBC-BGY
+LBC-PMI
+LBC-PSA
+LBC-NYO
+PIS-GRO
+PIS-STN
+VXO-ALC
+VXO-NRN
+WRO-ALC
+WRO-BLQ
+WRO-BRS
+WRO-CRL
+WRO-DUB
+WRO-NRN
+WRO-EMA
+WRO-HHN
+WRO-GRO
+WRO-PIK
+WRO-LPL
+WRO-STN
+WRO-AGP
+WRO-BGY
+WRO-RYG
+WRO-CIA
+WRO-SNN
+LPP-CRL
+LPP-NRN
+LPP-BGY
+MJV-BHX
+MJV-BOH
+MJV-DUB
+MJV-EMA
+MJV-PIK
+MJV-LBA
+MJV-LPL
+MJV-LTN
+MJV-STN
+LPL-ALC
+LPL-EGC
+LPL-BTS
+LPL-BRE
+LPL-CCF
+LPL-ORK
+LPL-LDY
+LPL-DUB
+LPL-FAO
+LPL-FUE
+LPL-GRO
+LPL-LPA
+LPL-IBZ
+LPL-KUN
+LPL-NOC
+LPL-KRK
+LPL-ACE
+LPL-LIG
+LPL-LCJ
+LPL-AGP
+LPL-BGY
+LPL-MJV
+LPL-FNI
+LPL-RYG
+LPL-TRF
+LPL-PMI
+LPL-PSA
+LPL-OPO
+LPL-POZ
+LPL-REU
+LPL-RIX
+LPL-RMI
+LPL-CIA
+LPL-SZG
+LPL-SVQ
+LPL-SNN
+LPL-NYO
+LPL-SZZ
+LPL-TFS
+LPL-TLN
+LPL-WRO
+TPS-AOI
+TPS-BRI
+TPS-BLL
+TPS-BLQ
+TPS-BTS
+TPS-CRL
+TPS-CAG
+TPS-CUF
+TPS-DUB
+TPS-NRN
+TPS-EIN
+TPS-HHN
+TPS-GOA
+TPS-GRO
+TPS-GSE
+TPS-IBZ
+TPS-FKB
+TPS-KRK
+TPS-LTN
+TPS-STN
+TPS-MST
+TPS-MAD
+TPS-MLA
+TPS-FMM
+TPS-BGY
+TPS-MUN
+TPS-TRF
+TPS-BVA
+TPS-PMF
+TPS-PEG
+TPS-PSR
+TPS-PSA
+TPS-CIA
+TPS-NYO
+TPS-TRS
+TPS-TRN
+TPS-VLC
+TPS-TSF
+TPS-VRN
+HAU-ALC
+HAU-BRE
+HAU-STN
+HAU-AGP
+LPA-BCN
+LPA-BHX
+LPA-BOH
+LPA-BTS
+LPA-BRE
+LPA-BRS
+LPA-CRL
+LPA-ORK
+LPA-DUB
+LPA-NRN
+LPA-EMA
+LPA-EDI
+LPA-EIN
+LPA-HHN
+LPA-PIK
+LPA-FKB
+LPA-NOC
+LPA-LPL
+LPA-LTN
+LPA-STN
+LPA-MAD
+LPA-CSO
+LPA-BGY
+LPA-PSA
+LPA-OPO
+LPA-SDR
+LPA-SCQ
+LPA-SVQ
+LPA-SNN
+LPA-VLC
+LPA-ZAZ
+SVQ-ALC
+SVQ-BCN
+SVQ-BRI
+SVQ-BLQ
+SVQ-BRS
+SVQ-CRL
+SVQ-CAG
+SVQ-DUB
+SVQ-NRN
+SVQ-EMA
+SVQ-EIN
+SVQ-FEZ
+SVQ-GRO
+SVQ-LPA
+SVQ-ACE
+SVQ-LPL
+SVQ-LGW
+SVQ-STN
+SVQ-MLA
+SVQ-RAK
+SVQ-MRS
+SVQ-BGY
+SVQ-PMO
+SVQ-PMI
+SVQ-BVA
+SVQ-PSA
+SVQ-REU
+SVQ-CIA
+SVQ-SCQ
+SVQ-TFS
+SVQ-VLC
+SVQ-TSF
+SVQ-ZAZ
+SNN-FUE
+SNN-LPA
+SNN-ACE
+SNN-LPL
+SNN-LGW
+SNN-STN
+SNN-AGP
+SNN-NTE
+SNN-PMI
+SNN-TFS
+SNN-WRO
+KIR-ALC
+KIR-DUB
+KIR-FAO
+KIR-HHN
+KIR-LTN
+KIR-STN
+SFT-GRO
+AHO-AOI
+AHO-BLL
+AHO-BTS
+AHO-DUB
+AHO-NRN
+AHO-EIN
+AHO-HHN
+AHO-GRO
+AHO-GSE
+AHO-STN
+AHO-MAD
+AHO-FMM
+AHO-BGY
+AHO-MUN
+AHO-TRF
+AHO-PMF
+AHO-PSA
+AHO-CIA
+AHO-NYO
+AHO-TSF
+AHO-VRN
+BVA-ALC
+BVA-BCN
+BVA-BRI
+BVA-BLQ
+BVA-BTS
+BVA-BDS
+BVA-CAG
+BVA-DUB
+BVA-EDI
+BVA-FAO
+BVA-FEZ
+BVA-GRO
+BVA-PIK
+BVA-GSE
+BVA-KUN
+BVA-KRK
+BVA-MAD
+BVA-AGP
+BVA-RAK
+BVA-BGY
+BVA-NDR
+BVA-RYG
+BVA-PMF
+BVA-PUF
+BVA-PSR
+BVA-PSA
+BVA-OPO
+BVA-REU
+BVA-CIA
+BVA-SDR
+BVA-SVQ
+BVA-NYO
+BVA-TNG
+BVA-TFS
+BVA-TPS
+BVA-TRN
+BVA-VLC
+BVA-TSF
+BVA-VRN
+BVA-ZAZ
+ZAZ-ALC
+ZAZ-BLQ
+ZAZ-CRL
+ZAZ-NRN
+ZAZ-LPA
+ZAZ-ACE
+ZAZ-STN
+ZAZ-AGP
+ZAZ-BGY
+ZAZ-BVA
+ZAZ-CIA
+ZAZ-SVQ
+ZAZ-TFS
+MAD-AHO
+MAD-ALC
+MAD-LEI
+MAD-AOI
+MAD-BRI
+MAD-BLQ
+MAD-CRL
+MAD-CAG
+MAD-DUB
+MAD-NRN
+MAD-EIN
+MAD-FAO
+MAD-FEZ
+MAD-HHN
+MAD-FUE
+MAD-GRO
+MAD-LPA
+MAD-IBZ
+MAD-XRY
+MAD-KRK
+MAD-ACE
+MAD-LGW
+MAD-STN
+MAD-MLA
+MAD-MAN
+MAD-RAK
+MAD-MRS
+MAD-BGY
+MAD-NDR
+MAD-RYG
+MAD-OUD
+MAD-PMI
+MAD-BVA
+MAD-PSA
+MAD-OPO
+MAD-CIA
+MAD-SDR
+MAD-SCQ
+MAD-NYO
+MAD-TNG
+MAD-TFS
+MAD-TPS
+MAD-TRN
+MAD-VLC
+MAD-VRN
+RHO-BGY
+RHO-PSA
+RHO-CIA
+ZAD-CRL
+ZAD-DUB
+ZAD-NRN
+ZAD-HHN
+ZAD-STN
+ZAD-RYG
+ZAD-NYO
+KID-GRO
+EMA-ALC
+EMA-EGC
+EMA-SXF
+EMA-CCF
+EMA-DNR
+EMA-DUB
+EMA-FAO
+EMA-FUE
+EMA-GRO
+EMA-LPA
+EMA-IBZ
+EMA-NOC
+EMA-KRK
+EMA-ACE
+EMA-LIG
+EMA-LCJ
+EMA-AGP
+EMA-RAK
+EMA-BGY
+EMA-MJV
+EMA-NTE
+EMA-PMI
+EMA-PSA
+EMA-REU
+EMA-RIX
+EMA-CIA
+EMA-RZE
+EMA-SVQ
+EMA-TLL
+EMA-TFS
+EMA-TRN
+EMA-VLC
+EMA-TSF
+EMA-WRO
+TSF-AHO
+TSF-ALC
+TSF-BCN
+TSF-BRI
+TSF-BRE
+TSF-BDS
+TSF-BRS
+TSF-CRL
+TSF-CAG
+TSF-DUB
+TSF-NRN
+TSF-EMA
+TSF-HHN
+TSF-GRO
+TSF-IBZ
+TSF-ACE
+TSF-LBA
+TSF-STN
+TSF-AGP
+TSF-MLA
+TSF-RYG
+TSF-PMO
+TSF-BVA
+TSF-REU
+TSF-SVQ
+TSF-NYO
+TSF-TPS
+TSF-VLC
+KSD-GRO
+FUE-BCN
+FUE-BHX
+FUE-BOH
+FUE-BRE
+FUE-CRL
+FUE-ORK
+FUE-DUB
+FUE-NRN
+FUE-EMA
+FUE-EDI
+FUE-HHN
+FUE-PIK
+FUE-LBA
+FUE-LPL
+FUE-LTN
+FUE-STN
+FUE-MAD
+FUE-BGY
+FUE-PSA
+FUE-SNN
+FUE-VLC
+BES-LTN
+BES-MRS
+CRL-AGA
+CRL-ALC
+CRL-AOI
+CRL-BCN
+CRL-BRI
+CRL-EGC
+CRL-BLQ
+CRL-BOD
+CRL-BTS
+CRL-BDS
+CRL-CAG
+CRL-CCF
+CRL-COR
+CRL-DUB
+CRL-EDI
+CRL-FAO
+CRL-FEZ
+CRL-FSC
+CRL-FUE
+CRL-GRO
+CRL-GSE
+CRL-LPA
+CRL-IBZ
+CRL-KUN
+CRL-KRK
+CRL-ACE
+CRL-LPP
+CRL-LCA
+CRL-LRH
+CRL-MAD
+CRL-AGP
+CRL-RAK
+CRL-MRS
+CRL-BGY
+CRL-MPL
+CRL-NDR
+CRL-FNI
+CRL-RYG
+CRL-OUD
+CRL-PMI
+CRL-PUF
+CRL-PGF
+CRL-PEG
+CRL-PSR
+CRL-PSA
+CRL-OPO
+CRL-REU
+CRL-RIX
+CRL-CIA
+CRL-SDR
+CRL-SVQ
+CRL-NYO
+CRL-TNG
+CRL-TFS
+CRL-TPS
+CRL-TRS
+CRL-TRN
+CRL-VLC
+CRL-VLL
+CRL-TSF
+CRL-VRN
+CRL-VOL
+CRL-WRO
+CRL-ZAD
+CRL-ZAZ
+VLL-ALC
+VLL-BCN
+VLL-CRL
+VLL-NRN
+VLL-ACE
+VLL-STN
+VLL-AGP
+VLL-BGY
+LDE-STN
+LDE-BGY
+VLC-BRI
+VLC-BLQ
+VLC-BRS
+VLC-CRL
+VLC-CAG
+VLC-DUB
+VLC-NRN
+VLC-EMA
+VLC-HHN
+VLC-FUE
+VLC-LPA
+VLC-IBZ
+VLC-ACE
+VLC-STN
+VLC-MAD
+VLC-AGP
+VLC-MLA
+VLC-RAK
+VLC-MRS
+VLC-FMM
+VLC-BGY
+VLC-MUN
+VLC-RYG
+VLC-PMI
+VLC-BVA
+VLC-PSA
+VLC-OPO
+VLC-CIA
+VLC-SCQ
+VLC-SVQ
+VLC-TFS
+VLC-TPS
+VLC-TRS
+VLC-TSF
+SZG-DUB
+SZG-PIK
+SZG-LPL
+SZG-STN
+OSI-HHN
+LDY-BHX
+LDY-FAO
+LDY-PIK
+LDY-LPL
+LDY-STN
+SCQ-ALC
+SCQ-BCN
+SCQ-HHN
+SCQ-LPA
+SCQ-ACE
+SCQ-STN
+SCQ-MAD
+SCQ-AGP
+SCQ-BGY
+SCQ-REU
+SCQ-SVQ
+SCQ-TFS
+SCQ-VLC
+HUY-ALC
+PUY-HHN
+PUY-STN
+CSO-ALC
+CSO-GRO
+CSO-LPA
+CSO-AGP
+TRN-BRI
+TRN-BDS
+TRN-CRL
+TRN-DUB
+TRN-EMA
+TRN-EDI
+TRN-GRO
+TRN-IBZ
+TRN-STN
+TRN-MAD
+TRN-BVA
+TRN-TPS
+GRZ-STN
+BLQ-ALC
+BLQ-BRI
+BLQ-BOD
+BLQ-BDS
+BLQ-CRL
+BLQ-DUB
+BLQ-NRN
+BLQ-EDI
+BLQ-FEZ
+BLQ-HHN
+BLQ-GRO
+BLQ-GRX
+BLQ-IBZ
+BLQ-KRK
+BLQ-SUF
+BLQ-ACE
+BLQ-LGW
+BLQ-STN
+BLQ-MAD
+BLQ-AGP
+BLQ-MLA
+BLQ-RAK
+BLQ-PMO
+BLQ-BVA
+BLQ-OPO
+BLQ-SVQ
+BLQ-TFS
+BLQ-TPS
+BLQ-VLC
+BLQ-WRO
+BLQ-ZAZ
+GRX-BLQ
+GRX-BGY
+TRF-AHO
+TRF-ALC
+TRF-BRE
+TRF-EDI
+TRF-HHN
+TRF-GRO
+TRF-LPL
+TRF-STN
+TRF-AGP
+TRF-BGY
+TRF-PSR
+TRF-PSA
+TRF-TPS
+BLL-AHO
+BLL-ALC
+BLL-EDI
+BLL-FAO
+BLL-GRO
+BLL-STN
+BLL-AGP
+BLL-MLA
+BLL-BGY
+BLL-PMI
+BLL-PSA
+BLL-CIA
+BLL-TFS
+BLL-TPS
+XRY-HHN
+XRY-STN
+XRY-MAD
+BDS-BLQ
+BDS-CRL
+BDS-EIN
+BDS-GRO
+BDS-STN
+BDS-BGY
+BDS-BVA
+BDS-PSA
+BDS-CIA
+BDS-NYO
+BDS-TRN
+BDS-TSF
+BDS-VRN
+GRO-AAR
+GRO-AHO
+GRO-BLL
+GRO-BHX
+GRO-BLQ
+GRO-BOH
+GRO-BTS
+GRO-BRE
+GRO-BDS
+GRO-BRS
+GRO-CRL
+GRO-DUB
+GRO-NRN
+GRO-EMA
+GRO-EIN
+GRO-FEZ
+GRO-HHN
+GRO-GDN
+GRO-PIK
+GRO-GSE
+GRO-LBC
+GRO-IBZ
+GRO-FKB
+GRO-KSD
+GRO-KUN
+GRO-KID
+GRO-ACE
+GRO-LCA
+GRO-LIL
+GRO-LPL
+GRO-LGW
+GRO-LTN
+GRO-STN
+GRO-MST
+GRO-MAD
+GRO-CSO
+GRO-MLA
+GRO-RAK
+GRO-FMM
+GRO-BGY
+GRO-MUN
+GRO-NCL
+GRO-TRF
+GRO-PMI
+GRO-BVA
+GRO-PEG
+GRO-PSR
+GRO-PSA
+GRO-PDV
+GRO-PIS
+GRO-POZ
+GRO-CIA
+GRO-RZE
+GRO-SVQ
+GRO-SFT
+GRO-NYO
+GRO-VST
+GRO-SZZ
+GRO-TLL
+GRO-TFS
+GRO-TPS
+GRO-TRN
+GRO-TSF
+GRO-WRO
+TRS-BHX
+TRS-CRL
+TRS-CAG
+TRS-NRN
+TRS-STN
+TRS-TPS
+TRS-VLC
+DSA-ALC
+DSA-FAO
+DSA-TFS
+RIX-BRE
+RIX-BRS
+RIX-CRL
+RIX-DUB
+RIX-NRN
+RIX-EMA
+RIX-HHN
+RIX-PIK
+RIX-LPL
+RIX-STN
+RIX-BGY
+RIX-RYG
+RIX-NYO
+RIX-TMP
+BTS-AHO
+BTS-ALC
+BTS-BHX
+BTS-BRS
+BTS-CRL
+BTS-DUB
+BTS-EDI
+BTS-GRO
+BTS-LPA
+BTS-LPL
+BTS-LTN
+BTS-STN
+BTS-AGP
+BTS-BGY
+BTS-PMI
+BTS-BVA
+BTS-CIA
+BTS-NYO
+BTS-TPS
+VOL-CRL
+VOL-HHN
+VOL-BGY
+VOL-CIA
+LRH-CRL
+LRH-ORK
+LRH-DUB
+LRH-STN
+LRH-RYG
+LRH-OPO
+MPL-CRL
+MPL-HHN
+MPL-LBA
+PSA-AHO
+PSA-ALC
+PSA-BRI
+PSA-BLL
+PSA-BOH
+PSA-BDS
+PSA-CRL
+PSA-CAG
+PSA-CND
+PSA-DUB
+PSA-NRN
+PSA-EMA
+PSA-EDI
+PSA-EIN
+PSA-FEZ
+PSA-HHN
+PSA-FUE
+PSA-GRO
+PSA-PIK
+PSA-GSE
+PSA-LPA
+PSA-LBC
+PSA-IBZ
+PSA-KRK
+PSA-SUF
+PSA-LBA
+PSA-LPL
+PSA-STN
+PSA-MST
+PSA-MAD
+PSA-AGP
+PSA-MLA
+PSA-RAK
+PSA-FMM
+PSA-MUN
+PSA-TRF
+PSA-PMO
+PSA-BVA
+PSA-OPO
+PSA-REU
+PSA-RHO
+PSA-SDR
+PSA-SVQ
+PSA-NYO
+PSA-TMP
+PSA-TFS
+PSA-TPS
+PSA-VLC
+TNG-CRL
+TNG-MAD
+TNG-MRS
+TNG-BGY
+TNG-BVA
+NOC-ALC
+NOC-BRS
+NOC-EMA
+NOC-FAO
+NOC-LPA
+NOC-ACE
+NOC-LBA
+NOC-LPL
+NOC-LTN
+NOC-STN
+NOC-REU
+NOC-TFS
+XCR-RYG
+XCR-OPO
+XCR-NYO
+TUF-DUB
+TUF-STN
+TUF-OPO
+LCA-CRL
+LCA-GRO
+PSR-CRL
+PSR-CAG
+PSR-EIN
+PSR-HHN
+PSR-GRO
+PSR-STN
+PSR-BGY
+PSR-TRF
+PSR-BVA
+PSR-TPS
+MAN-ALC
+MAN-DUB
+MAN-FAO
+MAN-MAD
+MAN-TFS
+LCJ-DUB
+LCJ-EMA
+LCJ-EDI
+LCJ-LPL
+LCJ-STN
+LCJ-BGY
+TMP-ALC
+TMP-BRE
+TMP-EDI
+TMP-HHN
+TMP-KUN
+TMP-STN
+TMP-AGP
+TMP-BGY
+TMP-RYG
+TMP-PSA
+TMP-RIX
+SUF-BLQ
+SUF-NRN
+SUF-BGY
+SUF-PSA
+BCN-CRL
+BCN-CAG
+BCN-DUB
+BCN-NRN
+BCN-EDI
+BCN-FUE
+BCN-LPA
+BCN-IBZ
+BCN-ACE
+BCN-LBA
+BCN-AGP
+BCN-BGY
+BCN-RYG
+BCN-PMI
+BCN-BVA
+BCN-OPO
+BCN-CIA
+BCN-SDR
+BCN-SCQ
+BCN-SVQ
+BCN-TFS
+BCN-VLL
+BCN-TSF
+FSC-CRL
+FSC-STN
+BZR-BRS
+BZR-NRN
+BZR-LTN
+BZR-RYG
+BZR-NYO
+ACE-BCN
+ACE-BHX
+ACE-BLQ
+ACE-BOH
+ACE-BRS
+ACE-CRL
+ACE-ORK
+ACE-DUB
+ACE-NRN
+ACE-EMA
+ACE-EDI
+ACE-EIN
+ACE-HHN
+ACE-GRO
+ACE-PIK
+ACE-FKB
+ACE-NOC
+ACE-LBA
+ACE-LPL
+ACE-LTN
+ACE-STN
+ACE-MAD
+ACE-SDR
+ACE-SCQ
+ACE-SVQ
+ACE-SNN
+ACE-VLC
+ACE-VLL
+ACE-TSF
+ACE-ZAZ
+REU-BHX
+REU-BRS
+REU-CRL
+REU-ORK
+REU-DUB
+REU-NRN
+REU-EMA
+REU-EIN
+REU-HHN
+REU-PIK
+REU-IBZ
+REU-NOC
+REU-KRK
+REU-LPL
+REU-LTN
+REU-STN
+REU-MST
+REU-RAK
+REU-FMM
+REU-BGY
+REU-MUN
+REU-NDR
+REU-PMI
+REU-BVA
+REU-PSA
+REU-SDR
+REU-SCQ
+REU-SVQ
+REU-TSF
+BZG-BHX
+BZG-DUB
+BZG-NRN
+BZG-STN
+KUN-ALC
+KUN-SXF
+KUN-BHX
+KUN-BRS
+KUN-CRL
+KUN-DUB
+KUN-NRN
+KUN-EDI
+KUN-HHN
+KUN-GRO
+KUN-GSE
+KUN-LPL
+KUN-LGW
+KUN-LTN
+KUN-STN
+KUN-BGY
+KUN-RYG
+KUN-BVA
+KUN-TMP
+PEG-CRL
+PEG-CAG
+PEG-GRO
+PEG-STN
+PEG-TPS
+LNZ-STN
+OUD-CRL
+OUD-MAD
+PMF-AHO
+PMF-CAG
+PMF-STN
+PMF-BVA
+PMF-TPS
+PMO-BLQ
+PMO-STN
+PMO-MRS
+PMO-BGY
+PMO-RYG
+PMO-PSA
+PMO-SVQ
+PMO-TFS
+PMO-TSF
+PMO-VRN
+PMI-ALC
+PMI-BCN
+PMI-BLL
+PMI-BHX
+PMI-BOH
+PMI-BTS
+PMI-BRE
+PMI-BRS
+PMI-CRL
+PMI-DUB
+PMI-NRN
+PMI-EMA
+PMI-EDI
+PMI-EIN
+PMI-HHN
+PMI-GRO
+PMI-PIK
+PMI-LBC
+PMI-LBA
+PMI-LPL
+PMI-STN
+PMI-MAD
+PMI-RYG
+PMI-REU
+PMI-SVQ
+PMI-SNN
+PMI-NYO
+PMI-VLC
+PDV-GRO
+PDV-STN
+DUB-ABZ
+DUB-AHO
+DUB-ALC
+DUB-BCN
+DUB-SXF
+DUB-BIQ
+DUB-BHX
+DUB-BLQ
+DUB-BTS
+DUB-BRS
+DUB-CRL
+DUB-BZG
+DUB-CCF
+DUB-ORK
+DUB-EMA
+DUB-EDI
+DUB-EIN
+DUB-FAO
+DUB-HHN
+DUB-FUE
+DUB-GDN
+DUB-GRO
+DUB-PIK
+DUB-LPA
+DUB-GNB
+DUB-IBZ
+DUB-KTW
+DUB-KUN
+DUB-KIR
+DUB-KRK
+DUB-ACE
+DUB-LRH
+DUB-LBA
+DUB-LPL
+DUB-LCJ
+DUB-LGW
+DUB-LTN
+DUB-STN
+DUB-MAD
+DUB-AGP
+DUB-MLA
+DUB-MAN
+DUB-FMM
+DUB-BGY
+DUB-MUN
+DUB-MJV
+DUB-NTE
+DUB-NCL
+DUB-NCE
+DUB-RYG
+DUB-PMI
+DUB-BVA
+DUB-PSA
+DUB-OPO
+DUB-POZ
+DUB-REU
+DUB-RIX
+DUB-RDZ
+DUB-CIA
+DUB-RZE
+DUB-SZG
+DUB-SDR
+DUB-SVQ
+DUB-NYO
+DUB-SZZ
+DUB-TLL
+DUB-TFS
+DUB-TUF
+DUB-TPS
+DUB-TRN
+DUB-VLC
+DUB-TSF
+DUB-WRO
+DUB-ZAD
+RDZ-DUB
+RDZ-STN
+RDZ-OPO
+KRK-ALC
+KRK-BHX
+KRK-BLQ
+KRK-CRL
+KRK-CAG
+KRK-DUB
+KRK-NRN
+KRK-EMA
+KRK-EDI
+KRK-LBA
+KRK-LPL
+KRK-STN
+KRK-MAD
+KRK-AGP
+KRK-MLA
+KRK-BGY
+KRK-RYG
+KRK-BVA
+KRK-PSA
+KRK-REU
+KRK-CIA
+KRK-NYO
+KRK-TPS
+FMM-AHO
+FMM-ALC
+FMM-BRE
+FMM-DUB
+FMM-EDI
+FMM-FAO
+FMM-GRO
+FMM-STN
+FMM-AGP
+FMM-RYG
+FMM-PSA
+FMM-OPO
+FMM-REU
+FMM-NYO
+FMM-TPS
+FMM-VLC
+DNR-EMA
+DNR-STN
+NDR-CRL
+NDR-MAD
+NDR-MRS
+NDR-BVA
+NDR-REU
+FEZ-ALC
+FEZ-BLQ
+FEZ-CRL
+FEZ-NRN
+FEZ-HHN
+FEZ-GRO
+FEZ-STN
+FEZ-MAD
+FEZ-MRS
+FEZ-BGY
+FEZ-BVA
+FEZ-PSA
+FEZ-CIA
+FEZ-SVQ
+LTN-BZR
+LTN-BTS
+LTN-BES
+LTN-DUB
+LTN-FUE
+LTN-GRO
+LTN-LPA
+LTN-KUN
+LTN-KIR
+LTN-NOC
+LTN-ACE
+LTN-MLA
+LTN-RAK
+LTN-MJV
+LTN-FNI
+LTN-REU
+LTN-RZE
+LTN-TLL
+LTN-TFS
+LTN-TPS
+TLL-BRE
+TLL-DUB
+TLL-NRN
+TLL-EMA
+TLL-EDI
+TLL-GRO
+TLL-LTN
+TLL-BGY
+TLL-RYG
+TLL-NYO
+RMI-HHN
+RMI-LPL
+RMI-STN
+RMI-NYO
+LEI-NRN
+LEI-STN
+LEI-MAD
+PUF-CRL
+PUF-STN
+PUF-BVA
+EDI-ALC
+EDI-BCN
+EDI-SXF
+EDI-BLL
+EDI-BLQ
+EDI-BOD
+EDI-BTS
+EDI-BRE
+EDI-CRL
+EDI-DUB
+EDI-NRN
+EDI-FAO
+EDI-HHN
+EDI-FUE
+EDI-GDN
+EDI-GSE
+EDI-LPA
+EDI-KUN
+EDI-KRK
+EDI-ACE
+EDI-LCJ
+EDI-AGP
+EDI-MLA
+EDI-RAK
+EDI-MRS
+EDI-FMM
+EDI-MUN
+EDI-TRF
+EDI-PMI
+EDI-BVA
+EDI-PSA
+EDI-POZ
+EDI-CIA
+EDI-NYO
+EDI-TLL
+EDI-TMP
+EDI-TFS
+EDI-TRN
+BRS-ALC
+BRS-EGC
+BRS-BZR
+BRS-BTS
+BRS-DUB
+BRS-FAO
+BRS-GDN
+BRS-GRO
+BRS-LPA
+BRS-IBZ
+BRS-KUN
+BRS-NOC
+BRS-ACE
+BRS-LIG
+BRS-AGP
+BRS-MLA
+BRS-RAK
+BRS-BGY
+BRS-PMI
+BRS-POZ
+BRS-REU
+BRS-RIX
+BRS-RZE
+BRS-SVQ
+BRS-TFS
+BRS-VLC
+BRS-TSF
+BRS-WRO
+ALC-BLL
+ALC-BHX
+ALC-BLQ
+ALC-BOH
+ALC-BTS
+ALC-BRE
+ALC-BRS
+ALC-CRL
+ALC-ORK
+ALC-DSA
+ALC-DUB
+ALC-NRN
+ALC-EMA
+ALC-EDI
+ALC-EIN
+ALC-FEZ
+ALC-HHN
+ALC-GDN
+ALC-PIK
+ALC-GSE
+ALC-HAU
+ALC-HUY
+ALC-FKB
+ALC-KUN
+ALC-KIR
+ALC-NOC
+ALC-KRK
+ALC-LBA
+ALC-LPL
+ALC-LGW
+ALC-STN
+ALC-MST
+ALC-MAD
+ALC-CSO
+ALC-MAN
+ALC-RAK
+ALC-FMM
+ALC-BGY
+ALC-MUN
+ALC-RYG
+ALC-TRF
+ALC-PMI
+ALC-BVA
+ALC-PSA
+ALC-POZ
+ALC-SDR
+ALC-SCQ
+ALC-SVQ
+ALC-NYO
+ALC-VST
+ALC-TMP
+ALC-VLL
+ALC-VXO
+ALC-TSF
+ALC-WRO
+ALC-ZAZ
+BRQ-STN
+BRQ-BGY
+VRN-AHO
+VRN-BDS
+VRN-CRL
+VRN-STN
+VRN-MAD
+VRN-PMO
+VRN-BVA
+VRN-TPS
+ORK-ALC
+ORK-BOD
+ORK-CCF
+ORK-DUB
+ORK-FAO
+ORK-FUE
+ORK-LPA
+ORK-ACE
+ORK-LRH
+ORK-LPL
+ORK-LGW
+ORK-STN
+ORK-AGP
+ORK-BGY
+ORK-REU
+ORK-TFS
+BRI-BLQ
+BRI-CRL
+BRI-CAG
+BRI-NRN
+BRI-HHN
+BRI-GOA
+BRI-FKB
+BRI-STN
+BRI-MAD
+BRI-MLA
+BRI-BGY
+BRI-BVA
+BRI-PSA
+BRI-CIA
+BRI-SVQ
+BRI-TPS
+BRI-TRN
+BRI-VLC
+BRI-TSF
+BRE-ALC
+BRE-EDI
+BRE-FAO
+BRE-FUE
+BRE-GDN
+BRE-GRO
+BRE-LPA
+BRE-HAU
+BRE-IBZ
+BRE-LPL
+BRE-STN
+BRE-AGP
+BRE-RAK
+BRE-FMM
+BRE-BGY
+BRE-MUN
+BRE-TRF
+BRE-PMI
+BRE-OPO
+BRE-RIX
+BRE-NYO
+BRE-TLL
+BRE-TMP
+BRE-TFS
+BRE-TSF
+NTE-DUB
+NTE-EMA
+NTE-LBA
+NTE-MRS
+NTE-SNN
+GDN-ALC
+GDN-BHX
+GDN-BRE
+GDN-BRS
+GDN-DUB
+GDN-EDI
+GDN-HHN
+GDN-GRO
+GDN-LBA
+GDN-STN
+GDN-RYG
+CIA-AHO
+CIA-BCN
+CIA-BRI
+CIA-BLL
+CIA-BTS
+CIA-BDS
+CIA-CRL
+CIA-CAG
+CIA-DUB
+CIA-NRN
+CIA-EMA
+CIA-EDI
+CIA-EIN
+CIA-FEZ
+CIA-HHN
+CIA-GRO
+CIA-PIK
+CIA-GSE
+CIA-IBZ
+CIA-FKB
+CIA-KRK
+CIA-LPL
+CIA-LGW
+CIA-STN
+CIA-MAD
+CIA-RAK
+CIA-MRS
+CIA-BGY
+CIA-RYG
+CIA-BVA
+CIA-OPO
+CIA-RHO
+CIA-SDR
+CIA-SVQ
+CIA-NYO
+CIA-TPS
+CIA-VLC
+CIA-VOL
+CIA-WRO
+CIA-ZAZ
+MST-ALC
+MST-FAO
+MST-GRO
+MST-AGP
+MST-PSA
+MST-OPO
+MST-REU
+MST-TFS
+MST-TPS
+GNB-DUB
+GNB-STN
+GNB-NYO
+IBZ-BCN
+IBZ-BHX
+IBZ-BLQ
+IBZ-BOH
+IBZ-BRE
+IBZ-BRS
+IBZ-CRL
+IBZ-CAG
+IBZ-DUB
+IBZ-NRN
+IBZ-EMA
+IBZ-EIN
+IBZ-HHN
+IBZ-GRO
+IBZ-PIK
+IBZ-LBA
+IBZ-LPL
+IBZ-STN
+IBZ-MAD
+IBZ-AGP
+IBZ-BGY
+IBZ-RYG
+IBZ-PSA
+IBZ-REU
+IBZ-CIA
+IBZ-NYO
+IBZ-TPS
+IBZ-TRN
+IBZ-VLC
+IBZ-TSF
+AAR-GRO
+AAR-STN
+AAR-AGP
+AAR-RYG
+CAG-BCN
+CAG-BRI
+CAG-CRL
+CAG-CUF
+CAG-NRN
+CAG-HHN
+CAG-GOA
+CAG-IBZ
+CAG-FKB
+CAG-KRK
+CAG-MAD
+CAG-MRS
+CAG-BGY
+CAG-BVA
+CAG-PMF
+CAG-PEG
+CAG-PSR
+CAG-PSA
+CAG-CIA
+CAG-SVQ
+CAG-TPS
+CAG-TRS
+CAG-VLC
+CAG-TSF
+EIN-AHO
+EIN-ALC
+EIN-BDS
+EIN-DUB
+EIN-FAO
+EIN-GRO
+EIN-LPA
+EIN-IBZ
+EIN-ACE
+EIN-STN
+EIN-MAD
+EIN-AGP
+EIN-MLA
+EIN-MRS
+EIN-BGY
+EIN-PMI
+EIN-PSR
+EIN-PSA
+EIN-OPO
+EIN-REU
+EIN-CIA
+EIN-SVQ
+EIN-NYO
+EIN-TPS
+PGF-CRL
+PGF-STN
+POZ-ALC
+POZ-BRS
+POZ-DUB
+POZ-EDI
+POZ-GRO
+POZ-LPL
+POZ-STN
+POZ-BGY
+EBU-OPO
+FAO-BLL
+FAO-BHX
+FAO-BOH
+FAO-BRE
+FAO-BRS
+FAO-CRL
+FAO-ORK
+FAO-LDY
+FAO-DSA
+FAO-DUB
+FAO-NRN
+FAO-EMA
+FAO-EDI
+FAO-EIN
+FAO-HHN
+FAO-PIK
+FAO-KIR
+FAO-NOC
+FAO-LBA
+FAO-LPL
+FAO-LGW
+FAO-STN
+FAO-MST
+FAO-MAD
+FAO-MAN
+FAO-FMM
+FAO-MUN
+FAO-RYG
+FAO-BVA
+FAO-OPO
+FAO-NYO
+BIQ-DUB
+BIQ-STN
+BIQ-NYO
+RZE-BHX
+RZE-BRS
+RZE-DUB
+RZE-EMA
+RZE-GRO
+RZE-LTN
+RZE-STN
+MLA-BRI
+MLA-BLL
+MLA-BHX
+MLA-BLQ
+MLA-BRS
+MLA-DUB
+MLA-EDI
+MLA-EIN
+MLA-GRO
+MLA-KRK
+MLA-LBA
+MLA-LTN
+MLA-MAD
+MLA-MRS
+MLA-PSA
+MLA-SVQ
+MLA-NYO
+MLA-TPS
+MLA-VLC
+MLA-TSF
+STN-AAR
+STN-AGA
+STN-AHO
+STN-ALC
+STN-LEI
+STN-AOC
+STN-AOI
+STN-BRI
+STN-EGC
+STN-SXF
+STN-BIQ
+STN-BLL
+STN-BLQ
+STN-BTS
+STN-BRE
+STN-BDS
+STN-BRQ
+STN-BZG
+STN-CCF
+STN-ORK
+STN-COR
+STN-CUF
+STN-LDY
+STN-DNR
+STN-DUB
+STN-NRN
+STN-EIN
+STN-FAO
+STN-FEZ
+STN-FSC
+STN-HHN
+STN-FUE
+STN-GDN
+STN-GOA
+STN-GRO
+STN-PIK
+STN-GSE
+STN-LPA
+STN-GRZ
+STN-GNB
+STN-LBC
+STN-HAU
+STN-IBZ
+STN-XRY
+STN-FKB
+STN-KTW
+STN-KUN
+STN-KIR
+STN-KLU
+STN-NOC
+STN-KRK
+STN-ACE
+STN-LRH
+STN-LIG
+STN-LNZ
+STN-LCJ
+STN-LDE
+STN-MAD
+STN-AGP
+STN-RAK
+STN-MRS
+STN-FMM
+STN-BGY
+STN-MUN
+STN-MJV
+STN-RYG
+STN-TRF
+STN-PMO
+STN-PMI
+STN-PMF
+STN-PUF
+STN-PGF
+STN-PEG
+STN-PSR
+STN-PSA
+STN-PDV
+STN-PIS
+STN-OPO
+STN-POZ
+STN-PUY
+STN-REU
+STN-RIX
+STN-RMI
+STN-RDZ
+STN-CIA
+STN-RZE
+STN-SZG
+STN-SDR
+STN-SCQ
+STN-SVQ
+STN-SNN
+STN-NYO
+STN-VST
+STN-SZZ
+STN-TMP
+STN-TFS
+STN-TLN
+STN-TUF
+STN-TPS
+STN-TRS
+STN-TRN
+STN-VLC
+STN-VLL
+STN-TSF
+STN-VRN
+STN-WRO
+STN-ZAD
+STN-ZAZ
+KTW-BHX
+KTW-DUB
+KTW-STN
+GOA-BRI
+GOA-CAG
+GOA-STN
+GOA-TPS
+FKB-ALC
+FKB-BRI
+FKB-CAG
+FKB-GRO
+FKB-LPA
+FKB-ACE
+FKB-STN
+FKB-OPO
+FKB-CIA
+FKB-NYO
+FKB-TFS
+FKB-TPS
+KLU-HHN
+KLU-GSE
+KLU-STN
+KLU-NYO
+CND-BGY
+CND-PSA
+ABZ-DUB
+MRS-AGA
+MRS-BES
+MRS-CRL
+MRS-CAG
+MRS-NRN
+MRS-EDI
+MRS-EIN
+MRS-FEZ
+MRS-GSE
+MRS-LIL
+MRS-STN
+MRS-MAD
+MRS-AGP
+MRS-MLA
+MRS-RAK
+MRS-NDR
+MRS-NTE
+MRS-PMO
+MRS-OPO
+MRS-CIA
+MRS-SVQ
+MRS-NYO
+MRS-TNG
+MRS-VLC
+KGS-HHN
+KGS-BGY
+OPO-BCN
+OPO-BLQ
+OPO-BOD
+OPO-BRE
+OPO-CRL
+OPO-CCF
+OPO-DUB
+OPO-NRN
+OPO-EIN
+OPO-FAO
+OPO-HHN
+OPO-LPA
+OPO-FKB
+OPO-LRH
+OPO-LIL
+OPO-LPL
+OPO-LGW
+OPO-STN
+OPO-MST
+OPO-MAD
+OPO-RAK
+OPO-MRS
+OPO-FMM
+OPO-BGY
+OPO-MUN
+OPO-BVA
+OPO-XCR
+OPO-PSA
+OPO-RDZ
+OPO-CIA
+OPO-EBU
+OPO-TFS
+OPO-TUF
+OPO-VLC
+VST-ALC
+VST-GRO
+VST-STN
+VST-AGP
+NRN-AGA
+NRN-AHO
+NRN-ALC
+NRN-LEI
+NRN-AOI
+NRN-BCN
+NRN-BRI
+NRN-BZR
+NRN-BHX
+NRN-BLQ
+NRN-BZG
+NRN-CAG
+NRN-EDI
+NRN-FAO
+NRN-FEZ
+NRN-FUE
+NRN-GRO
+NRN-GSE
+NRN-LPA
+NRN-IBZ
+NRN-KUN
+NRN-KRK
+NRN-SUF
+NRN-ACE
+NRN-LPP
+NRN-LBA
+NRN-STN
+NRN-MAD
+NRN-AGP
+NRN-RAK
+NRN-MRS
+NRN-BGY
+NRN-RYG
+NRN-PMI
+NRN-PSA
+NRN-OPO
+NRN-REU
+NRN-RIX
+NRN-CIA
+NRN-SVQ
+NRN-NYO
+NRN-TLL
+NRN-TFS
+NRN-TPS
+NRN-TRS
+NRN-VLC
+NRN-VLL
+NRN-VXO
+NRN-TSF
+NRN-WRO
+NRN-ZAD
+NRN-ZAZ
+SDR-ALC
+SDR-BCN
+SDR-CRL
+SDR-DUB
+SDR-HHN
+SDR-LPA
+SDR-ACE
+SDR-STN
+SDR-MAD
+SDR-AGP
+SDR-BGY
+SDR-BVA
+SDR-PSA
+SDR-REU
+SDR-CIA
+SDR-TFS
+CUF-CAG
+CUF-STN
+CUF-TPS
+HHN-AGA
+HHN-AHO
+HHN-ALC
+HHN-BRI
+HHN-BLQ
+HHN-CAG
+HHN-DUB
+HHN-EDI
+HHN-FAO
+HHN-FEZ
+HHN-FUE
+HHN-GDN
+HHN-GRO
+HHN-GSE
+HHN-LPA
+HHN-IBZ
+HHN-XRY
+HHN-KUN
+HHN-KIR
+HHN-KLU
+HHN-KGS
+HHN-ACE
+HHN-ILD
+HHN-STN
+HHN-MAD
+HHN-AGP
+HHN-RAK
+HHN-BGY
+HHN-MPL
+HHN-OSI
+HHN-TRF
+HHN-PMI
+HHN-PSR
+HHN-PSA
+HHN-OPO
+HHN-PUY
+HHN-REU
+HHN-RIX
+HHN-RMI
+HHN-CIA
+HHN-SDR
+HHN-SCQ
+HHN-NYO
+HHN-TMP
+HHN-TFS
+HHN-TPS
+HHN-VLC
+HHN-TSF
+HHN-VOL
+HHN-WRO
+HHN-ZAD
+BHX-ALC
+BHX-BTS
+BHX-BZG
+BHX-LDY
+BHX-DUB
+BHX-NRN
+BHX-FAO
+BHX-FUE
+BHX-GDN
+BHX-GRO
+BHX-LPA
+BHX-IBZ
+BHX-KTW
+BHX-KUN
+BHX-KRK
+BHX-ACE
+BHX-AGP
+BHX-MLA
+BHX-MJV
+BHX-PMI
+BHX-REU
+BHX-RZE
+BHX-TFS
+BHX-TRS
+NCL-DUB
+NCL-GRO
+NCL-RYG
+NCE-DUB
+LGW-ALC
+LGW-BLQ
+LGW-ORK
+LGW-DUB
+LGW-FAO
+LGW-GRO
+LGW-KUN
+LGW-MAD
+LGW-BGY
+LGW-RYG
+LGW-OPO
+LGW-CIA
+LGW-SVQ
+LGW-SNN
+LGW-NYO
+TFS-BCN
+TFS-BLL
+TFS-BHX
+TFS-BLQ
+TFS-BOH
+TFS-BRE
+TFS-BRS
+TFS-CRL
+TFS-ORK
+TFS-DSA
+TFS-DUB
+TFS-NRN
+TFS-EMA
+TFS-EDI
+TFS-HHN
+TFS-GRO
+TFS-PIK
+TFS-FKB
+TFS-NOC
+TFS-LPL
+TFS-LTN
+TFS-STN
+TFS-MST
+TFS-MAD
+TFS-MAN
+TFS-BGY
+TFS-PMO
+TFS-BVA
+TFS-PSA
+TFS-OPO
+TFS-SDR
+TFS-SCQ
+TFS-SVQ
+TFS-SNN
+TFS-VLC
+TFS-ZAZ
49 download_dests.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+try:
+ import json # 2.6 and above
+except ImportError:
+ import simplejson as json # 2.5 and below
+
+import httplib, re, logging
+
+def get_dests(logger=None):
+ if logger is None:
+ logging.basicConfig(level=logging.DEBUG)
+ logger = logging.getLogger('RyanairDest')
+
+ conn = httplib.HTTPConnection('www.ryanair.com')
+ logger.debug("Sending request for destinations array")
+ conn.request("GET", '/en/booking/form')
+ resp = conn.getresponse()
+ logger.info("Request success, response code: %d" % resp.status)
+ if (resp.status != 200):
+ raise RyanairException("Invalid return status from Ryanair. \
+ Expected: 200, got: %d" % resp.status)
+ html = resp.read()
+ airport_part = re.search('var Dests=.*\s*([\S\s]*?);', html)
+ if airport_part is None:
+ raise RyanairException("Parsing of ryanair airports changed.")
+
+ airport_arr = re.split(",\n", airport_part.group(1))
+ logger.info("%d airports found" % int(len(airport_arr)/2))
+ airport_dirs, airport_names = {}, {}
+ for it in airport_arr:
+ # Example: sKUN='Kaunas'
+ if it[0] == 's':
+ airport_names[it[1:4]] = it[6:-1]
+ # Example line: aKUN=aKUN=['STN','BGY','RYG','BVA','TMP']
+ else:
+ short, dirs = it[1:4], it[5:]
+ # Expand dirs (this looks like json array, but improper)
+ airport_dirs[short] = json.loads(dirs.replace("'", '"'))
+ return airport_names, airport_dirs
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.ERROR)
+ logger = logging.getLogger('RyanairDest')
+
+ names, dests = get_dests(logger)
+ for k, v in dests.iteritems():
+ for dest in v:
+ print ("%s-%s" % (k, dest) )
126 go_1crawler
@@ -0,0 +1,126 @@
+#!/bin/sh
+export WHOAMI=`basename "$0"`
+HERE=`pwd`
+ACTIONBASEDIR="$HERE/actiondir"
+DATADIR="$HERE/datadir"
+NUMBER="$1"
+
+function usage() {
+ echo -n "Enter one numeric agument between 1 and "
+ wc -l < tunnel_hosts.txt
+ exit 1
+}
+
+function log() {
+ echo `date "+%F+%T" | tr -d '\n'` $WHOAMI[$$] "$1" >> $LOGFILE
+}
+
+function fail() {
+ if [ -n "$1" ]; then
+ log "----------------------------------------------------------"\
+ "$1" \
+ "----------------------------------------------------------"
+ fi
+
+ if [ -n "$DEST" ]; then # Adding the current dest for processing
+ ./next_dest "$DEST"
+ fi
+
+ if [ -f "$AUTOSSH_PIDFILE" ]; then
+ log "Attempting to kill $AUTOSSH_PIDFILE"
+ kill `cat $AUTOSSH_PIDFILE`
+ fi
+
+ echo "$1" >> $ACTIONBASEDIR/fail-$NUMBER.log
+ if [ -n "$FBASE" ]; then
+ touch "$FBASE.fail"
+ fi
+ exit 1
+}
+
+function proc_params() {
+ [[ -n "$(echo "$NUMBER" | grep -E "^[0-9]+$")" ]] || usage
+
+ SOX="$(sed -n "${NUMBER}p" tunnel_hosts.txt)"
+ [[ -n "$SOX" ]] || usage
+
+ export LOGFILE="$ACTIONBASEDIR/app-$NUMBER.log"
+
+ export HOST="$(echo $SOX | cut -d: -f1)"
+ export PORT="$(echo $SOX | cut -d: -f2)"
+
+ export SPORT=$((10000+$NUMBER)) # Socks port
+ export CPORT=$((22000+$NUMBER*29-5)) # Check port
+}
+
+function check_if_sport_used() {
+ if [ -n "`lsof -i TCP:${SPORT}`" ]; then
+ fail "Port ${SPORT} is busy:\n`lsof -i TCP:${SPORT}`"
+ fi
+}
+
+function start_autossh() {
+ export AUTOSSH_LOGFILE=$LOGFILE
+ export AUTOSSH_PIDFILE=$ACTIONBASEDIR/autossh-$NUMBER.pid
+
+ autossh -M${CPORT} -f -ND${SPORT} -i${HERE}/crawler_key -p${PORT} ${HOST}
+ RETCODE=$?
+ if [ "$RETCODE" != "0" ]; then
+ fail "General autossh fail (ret code = $RETCODE).\n"\
+ "Please have a look at $AUTOSSH_LOGFILE"
+ fi
+}
+
+function check_autossh_pid() {
+ if [ ! -f "$AUTOSSH_PIDFILE" ]; then
+ fail "Autossh pidfile '${AUTOSSH_PIDFILE}' not found"
+ fi
+}
+
+function check_if_display_ok() {
+ if [ -z "$DISPLAY" ]; then
+ fail "Bad display: $DISPLAY"
+ fi
+}
+
+
+[[ "$#" = 1 ]] || usage
+
+trap fail SIGINT SIGTERM
+
+proc_params
+check_if_sport_used
+check_if_display_ok
+start_autossh
+
+while true; do
+ check_autossh_pid
+
+ export DEST="$(./next_dest)"
+ TIME="$(date '+%F+%T')"
+ DATEHOUR=$(echo $TIME | perl -pe 's/(.*)\+(\d+).*/\1_\2/') # 2010-01-01_13
+
+ mkdir -p "${DATADIR}/${DATEHOUR}"
+ export FBASE="${DATADIR}/${DATEHOUR}/${DEST}_${TIME}-${NUMBER}"
+ touch "${FBASE}.started"
+
+ from="`echo $DEST | cut -d- -f1`"
+ to="`echo $DEST | cut -d- -f2`"
+ check_days=180
+ socks_host="127.0.0.1"
+ socks_port="$SPORT"
+ export from to check_days socks_host socks_port DISPLAY
+
+ java -jar ryanaid.jar > "$FBASE.flights" 2> "$FBASE.inf"
+
+ RET=$?
+ if [ "$RET" = "0" ]; then # all fine
+ rm "${FBASE}.started"
+ else
+ touch "${FBASE}.fail"
+ ./next_dest "${DEST}"
+ fi
+ export FBASE=
+
+ sleep 5
+done
19 next_dest
@@ -0,0 +1,19 @@
+#!/bin/sh
+lockfile -l 10 next_dest.lock
+
+if [ "$#" = "1" ]; then
+ sed -i "1i ${1}" next_dest.txt
+else
+ ret="$(head -1 next_dest.txt)"
+ [[ -z "$ret" ]] && ret="$(head -1 dests.txt)"
+
+ TMP="$(grep -v "$ret" < next_dest.txt)"
+
+ if [ -z "$TMP" ]; then # Add another item to next_dest.txt from dests
+ echo "$(grep -A1 "$ret" < dests.txt | grep -v "$ret" )" > next_dest.txt
+ else
+ echo "$TMP" > next_dest.txt
+ fi
+ echo $ret
+fi
+rm -f next_dest.lock
BIN  ryanaid.jar
Binary file not shown
BIN  ryanaid_lib/selenium-java-2.0a7-srcs.jar
Binary file not shown
BIN  ryanaid_lib/selenium-java-2.0a7.jar
Binary file not shown
BIN  ryanaid_lib/selenium-server-2.0a7-srcs.jar
Binary file not shown
BIN  ryanaid_lib/selenium-server-2.0a7.jar
Binary file not shown
BIN  ryanaid_lib/selenium-server-standalone-2.0a7.jar
Binary file not shown
49 src/Flight.java
@@ -0,0 +1,49 @@
+package ryanaid;
+
+import java.util.Calendar;
+import java.util.Scanner;
+import java.util.regex.MatchResult;
+
+public class Flight {
+ public Calendar departDate;
+ public Calendar returnTime;
+ public Calendar created;
+ public int seatsLeft = 0; // 0 = max
+ public String price;
+ public String from;
+ public String to;
+
+ public Flight(String from, String to, Calendar departDate, String returnTimeStr, String price, int seatsLeft) {
+ created = Calendar.getInstance();
+
+ this.departDate = (Calendar) departDate.clone();
+ returnTime = (Calendar) departDate.clone();
+ // returnTimeStr is HH:MM format
+ Scanner returnTime_sc = new Scanner(returnTimeStr);
+ returnTime_sc.findInLine("(\\d+):(\\d+)");
+ MatchResult rt = returnTime_sc.match();
+ returnTime.set(Calendar.HOUR_OF_DAY, Integer.parseInt(rt.group(1)));
+ returnTime.set(Calendar.MINUTE, Integer.parseInt(rt.group(2)));
+
+ if (returnTime.before(departDate)) {
+ // Lands after midnight or going east (time zone change).
+ // If time difference is more than 12 hours,
+ // then tomorrow
+ // else - do nothing.
+ long diff = departDate.getTimeInMillis() - returnTime.getTimeInMillis();
+ if (diff > 1000 * 60 * 60 * 12) {
+ returnTime.add(Calendar.DAY_OF_YEAR, 1);
+ }
+ }
+
+ // counting return time.
+ this.price = price;
+ this.seatsLeft = seatsLeft;
+ this.from = from;
+ this.to = to;
+ }
+ public String toString() {
+ return from+"-"+to+"|"+H.humanTime(departDate)+"|"+H.humanTime(returnTime)+
+ "|"+price+"|"+seatsLeft+"|"+H.humanTimeSec(created);
+ }
+}
110 src/Go1.java
@@ -0,0 +1,110 @@
+package ryanaid;
+
+import java.util.Calendar;
+import java.util.Scanner;
+import java.util.logging.*;
+import java.util.regex.MatchResult;
+
+import java.util.Map;
+
+public class Go1 {
+
+ private static Logger logger = Logger.getLogger("ryanair");
+ public static void main(String[] args) {
+ RyanairLogic crawler = null;
+ try {
+ logger.log(Level.INFO, "Starting Browser");
+
+ int wait_ms = 2000; // Wait after a click, ms
+ int check_days = 3650; // By default check for the next 10 years
+ //int check_days = 15; // By default check for the next 10 years
+
+ //String from = "Kaunas (KUN)";
+ //String to = "Edinburgh (EDI)";
+ String from = "KUN";
+ String to = "GRO";
+ //String to = "Frankfurt-Hahn (HHN)";
+
+ Integer socks_port = 0;
+ String socks_host = "";
+
+ Calendar date_from = Calendar.getInstance();
+ date_from.add(Calendar.DAY_OF_YEAR, 1); // start tomorrow by default
+
+ Map<String, String> env = System.getenv();
+
+ if (env.containsKey("date_from")) {
+ Scanner date_from_sc = new Scanner(env.get("date_from"));
+ date_from_sc.findInLine("(\\d+)-(\\d+)-(\\d+)");
+ MatchResult res = date_from_sc.match();
+ date_from.set(
+ Integer.parseInt(res.group(1)),
+ Integer.parseInt(res.group(2))+1,
+ Integer.parseInt(res.group(3))
+ );
+ }
+
+ if (env.containsKey("check_days")) check_days = Integer.parseInt(env.get("check_days"));
+ if (env.containsKey("from")) from = env.get("from");
+ if (env.containsKey("to")) to = env.get("to");
+ if (env.containsKey("wait_ms")) wait_ms = Integer.parseInt(env.get("wait_ms"));
+
+ Calendar dateTo = Calendar.getInstance();
+ dateTo.add(Calendar.DAY_OF_YEAR, check_days); // add this # of days
+ logger.log(Level.INFO, "Checking up to " + H.humanTime(dateTo));
+
+ if (env.containsKey("socks_host")) socks_host = env.get("socks_host");
+ if (env.containsKey("socks_port")) socks_port = Integer.parseInt(env.get("socks_port"));
+ try {
+ crawler = new RyanairLogic("FirefoxFirebug", socks_port, socks_host);
+ } catch (Exception e) {
+ logger.log(Level.INFO, "FirefoxFirebug crashed, opening w/o one");
+ crawler = new RyanairLogic("Firefox", socks_port, socks_host);
+ }
+ crawler.waitAfterClick = wait_ms; // Wait after clicking anything, ms
+ crawler.getFlights(from, to, date_from);
+
+ while (crawler.hasNextFlight()) {
+ Flight flight = null;
+ try {
+ flight = crawler.nextFlight();
+ } catch (PleaseFlightAgainException e) {
+ logger.log(Level.WARNING, "Something went wrong fetching the flight, fetching another one");
+ continue;
+ } catch (PleaseMakeNewSearchException e) {
+ logger.log(Level.INFO, e.getMessage());
+ crawler.getFlights();
+ continue;
+ } catch (PleaseNextWeekException e) {
+ crawler.next_week = true;
+ continue;
+ } catch (PleaseStopThisSearchException e) {
+ logger.log(Level.SEVERE, e.getMessage());
+ System.err.println(crawler.getCurrentUrl()+"\n\n"+crawler.getPageSource());
+ crawler.close();
+ System.exit(0);
+ }
+ System.out.println(flight);
+
+ if (crawler.date_now.after(dateTo)) {
+ logger.log(Level.INFO, "This was the last day we needed. Exiting");
+ crawler.close();
+ System.exit(0);
+ }
+ H.waitALittle(wait_ms);
+ }
+ } catch (Exception e) {
+ logger.log(Level.SEVERE, "Application crashed. Here is the trace");
+ e.printStackTrace();
+ logger.log(Level.SEVERE, "Here is the HTML:");
+ try {
+ System.err.println(crawler.getCurrentUrl()+"\n\n"+crawler.getPageSource());
+ } catch (Exception e2) {
+ logger.log(Level.SEVERE, "Printing page source failed. Something is REALLY wrong here.");
+ }
+ crawler.close();
+ System.exit(1);
+ }
+ crawler.close();
+ }
+}
26 src/H.java
@@ -0,0 +1,26 @@
+package ryanaid;
+
+import java.util.Calendar;
+
+public class H {
+ public static String twoDigitNum(int number) {
+ String number_str = number+"";