diff --git a/.gitignore b/.gitignore index d37d0c2cf4..bd2d5fbb88 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ cache/csv/*.csv cache/sitemap/*.xml portality/migrate/p1p2/*.csv portality/migrate/p1p2/*.csv.bak +/portality/scripts/3821_add_deposit_policy/add-Mirabel-deposit-policy.csv local_store/main/* !local_store/main/README.md local_store/tmp/* diff --git a/cms/assets/img/ambassadors/mahmoud-new.jpg b/cms/assets/img/ambassadors/mahmoud-new.jpg new file mode 100644 index 0000000000..cf543dc9e3 Binary files /dev/null and b/cms/assets/img/ambassadors/mahmoud-new.jpg differ diff --git a/cms/assets/img/ambassadors/mahmoud.jpg b/cms/assets/img/ambassadors/mahmoud.jpg deleted file mode 100644 index ee0dc409ab..0000000000 Binary files a/cms/assets/img/ambassadors/mahmoud.jpg and /dev/null differ diff --git a/cms/assets/img/sponsors/ALPSP.png b/cms/assets/img/sponsors/ALPSP.png new file mode 100644 index 0000000000..24aa343878 Binary files /dev/null and b/cms/assets/img/sponsors/ALPSP.png differ diff --git a/cms/assets/img/sponsors/SN_stack_logo.png b/cms/assets/img/sponsors/SN_stack_logo.png new file mode 100644 index 0000000000..ccb1864117 Binary files /dev/null and b/cms/assets/img/sponsors/SN_stack_logo.png differ diff --git a/cms/assets/img/volunteers/Agirre.JPG b/cms/assets/img/volunteers/Agirre.JPG new file mode 100644 index 0000000000..5f9d424444 Binary files /dev/null and b/cms/assets/img/volunteers/Agirre.JPG differ diff --git a/cms/assets/img/volunteers/Andista.jpg b/cms/assets/img/volunteers/Andista.jpg new file mode 100644 index 0000000000..2b8a8ebf18 Binary files /dev/null and b/cms/assets/img/volunteers/Andista.jpg differ diff --git a/cms/assets/img/volunteers/Carla.jpg b/cms/assets/img/volunteers/Carla.jpg new file mode 100644 index 0000000000..28215f7b5f Binary files /dev/null and b/cms/assets/img/volunteers/Carla.jpg differ diff --git a/cms/assets/img/volunteers/Costa.jpeg b/cms/assets/img/volunteers/Costa.jpeg new file mode 100644 index 0000000000..9a65e556b4 Binary files /dev/null and b/cms/assets/img/volunteers/Costa.jpeg differ diff --git a/cms/assets/img/volunteers/DEDI.jpg b/cms/assets/img/volunteers/DEDI.jpg new file mode 100644 index 0000000000..0108e42efc Binary files /dev/null and b/cms/assets/img/volunteers/DEDI.jpg differ diff --git a/cms/assets/img/volunteers/Danner..png b/cms/assets/img/volunteers/Danner..png new file mode 100644 index 0000000000..39db5a0ebd Binary files /dev/null and b/cms/assets/img/volunteers/Danner..png differ diff --git a/cms/assets/img/volunteers/Dr. K.JPG b/cms/assets/img/volunteers/Dr. K.JPG new file mode 100644 index 0000000000..4a9d254f72 Binary files /dev/null and b/cms/assets/img/volunteers/Dr. K.JPG differ diff --git a/cms/assets/img/volunteers/Flavius.jpg b/cms/assets/img/volunteers/Flavius.jpg new file mode 100644 index 0000000000..73a7ed257b Binary files /dev/null and b/cms/assets/img/volunteers/Flavius.jpg differ diff --git a/cms/assets/img/volunteers/Julio.jpg b/cms/assets/img/volunteers/Julio.jpg new file mode 100644 index 0000000000..10444ac27f Binary files /dev/null and b/cms/assets/img/volunteers/Julio.jpg differ diff --git a/cms/assets/img/volunteers/Khoirul.jpg b/cms/assets/img/volunteers/Khoirul.jpg new file mode 100644 index 0000000000..61925b8266 Binary files /dev/null and b/cms/assets/img/volunteers/Khoirul.jpg differ diff --git a/cms/assets/img/volunteers/Lee.jpg b/cms/assets/img/volunteers/Lee.jpg new file mode 100644 index 0000000000..ae76daf109 Binary files /dev/null and b/cms/assets/img/volunteers/Lee.jpg differ diff --git a/cms/assets/img/volunteers/Liping.jpg b/cms/assets/img/volunteers/Liping.jpg new file mode 100644 index 0000000000..f209327a08 Binary files /dev/null and b/cms/assets/img/volunteers/Liping.jpg differ diff --git a/cms/assets/img/volunteers/Longo.jpg b/cms/assets/img/volunteers/Longo.jpg new file mode 100644 index 0000000000..36aa8e8e87 Binary files /dev/null and b/cms/assets/img/volunteers/Longo.jpg differ diff --git a/cms/assets/img/volunteers/Marchitelli.jpg b/cms/assets/img/volunteers/Marchitelli.jpg new file mode 100644 index 0000000000..7676ba9943 Binary files /dev/null and b/cms/assets/img/volunteers/Marchitelli.jpg differ diff --git a/cms/assets/img/volunteers/Margalejo.jpeg b/cms/assets/img/volunteers/Margalejo.jpeg new file mode 100644 index 0000000000..8999172fc2 Binary files /dev/null and b/cms/assets/img/volunteers/Margalejo.jpeg differ diff --git a/cms/assets/img/volunteers/Martyna.JPG b/cms/assets/img/volunteers/Martyna.JPG new file mode 100644 index 0000000000..ffdd0fa604 Binary files /dev/null and b/cms/assets/img/volunteers/Martyna.JPG differ diff --git a/cms/assets/img/volunteers/Mirecka.jpeg b/cms/assets/img/volunteers/Mirecka.jpeg new file mode 100644 index 0000000000..338eda062e Binary files /dev/null and b/cms/assets/img/volunteers/Mirecka.jpeg differ diff --git a/cms/assets/img/volunteers/Octav.jpg b/cms/assets/img/volunteers/Octav.jpg new file mode 100644 index 0000000000..47657be109 Binary files /dev/null and b/cms/assets/img/volunteers/Octav.jpg differ diff --git a/cms/assets/img/volunteers/Penabad.png b/cms/assets/img/volunteers/Penabad.png new file mode 100644 index 0000000000..368087b146 Binary files /dev/null and b/cms/assets/img/volunteers/Penabad.png differ diff --git a/cms/assets/img/volunteers/Rasoul.jpg b/cms/assets/img/volunteers/Rasoul.jpg new file mode 100644 index 0000000000..ebb8a32f61 Binary files /dev/null and b/cms/assets/img/volunteers/Rasoul.jpg differ diff --git a/cms/assets/img/volunteers/Ratodi.jpg b/cms/assets/img/volunteers/Ratodi.jpg new file mode 100644 index 0000000000..41b2d21ddb Binary files /dev/null and b/cms/assets/img/volunteers/Ratodi.jpg differ diff --git a/cms/assets/img/volunteers/Wileidys.jpg b/cms/assets/img/volunteers/Wileidys.jpg new file mode 100644 index 0000000000..8c83fd4328 Binary files /dev/null and b/cms/assets/img/volunteers/Wileidys.jpg differ diff --git a/cms/assets/img/volunteers/Yang.jpg b/cms/assets/img/volunteers/Yang.jpg new file mode 100644 index 0000000000..5edb4f3656 Binary files /dev/null and b/cms/assets/img/volunteers/Yang.jpg differ diff --git a/cms/data/ambassadors.yml b/cms/data/ambassadors.yml index f2a949b4f0..db06e7db21 100644 --- a/cms/data/ambassadors.yml +++ b/cms/data/ambassadors.yml @@ -65,8 +65,8 @@ - name: Mahmoud Khalifa region: Middle East and Persian Gulf - bio: "Mahmoud is the founder of Cybrarians, the Arabic Portal for Librarianship and Information, a non-profit organization located in Egypt since 2002, and working on serving library and information community in Arab countries. He is PhD candidate in Humboldt University of Berlin, Germany. He works on a dissertation about The national information policies. Mahmoud was an Associate at IFLA International Leaders Programme, 2016-2018, and an accredited trainer for the IFLA BSLA programme (Building Strong Library Association), as well as an OCLC Fellow at Jay Jordan IFLA/OCLC Early Career Development Fellowship Program, class 2010. In 2013 he was the winner of the Scientific Research Academy’s prize in informatics in Egypt. Professionally, he worked in Library of Congress Cairo Office from 2006-2018." - photo: "mahmoud.jpg" + bio: "Mahmoud is a PhD candidate at Humboldt University of Berlin, Germany. He is writing his dissertation about national information policies. Mahmoud was an associate at the IFLA International Leaders Programme, 2016-2018, and an accredited trainer for the IFLA BSLA programme (Building Strong Library Association), as well as an OCLC Fellow at Jay Jordan IFLA/OCLC Early Career Development Fellowship Program, class 2010. Professionally, he worked in the Library of Congress Cairo Office from 2006-2018. As a researcher, he published more than 20 articles, book chapters, and conference papers." + photo: "mahmoud-new.jpg" coi: 2022: https://drive.google.com/file/d/10s7B0WeTPqpaafhThv-i8q03uIFaAAue/view?usp=sharing @@ -127,13 +127,6 @@ He has been an editor of Journal of Educational Evaluation for Health Profession photo: "sun-huh.jpg" coi: -- name: Thomas Herve Mboa Nkoudou - region: West and Central Africa - bio: "Thomas is a PhD candidate in Public Communication at Université Laval (Canada) and deeply engaged in advocating for open science and promoting best practices of scholarly communication in Africa. He has been involved in promoting open access in Haiti and French-speaking African countries (projet SOHA) and is the initiator and organiser of the yearly Africa Open Science and Hardware Summit (AfricaOSH). He is a member of the OpenCon organising committee and an instructor at Force11 Scholarly Communication Institute. He has made it his mission to make scientific resources produced in African-based universities more visible locally and abroad." - photo: "thomas_mboa.png" - coi: - 2022: https://drive.google.com/file/d/19rmrksWpCdP_GAHH8ZPT-HkaBgIjwA1x/view?usp=sharing - - name: Vrushali Dandawate region: India bio: "Vrushali is Head Librarian at AISSMS College of Engineering College Pune, Maharashtra, India, and She holds a Ph.D. degree from Reva University Banglore, India. Her research topic is 'OPEN ACCESS E -RESOURCES DEVELOPMENT IN ASIA: A STUDY' Vrushali was the winner of the ALCTS Online Course Grant for Library Professionals from Developing Countries in 2014, and the INASP open access week competition in both 2015 and 2016. She was invited to OpenCon 2017 in Berlin, Germany, and subsequently worked as a member of the organizing committee for OpenCon 2018 in Toronto, Canada. She also served as an advisory committee member for Open Access Week 2018-2019. She is selected as an advisory committee member of OpenDOAR and conference committee member of the FORCE2021 Conference." diff --git a/cms/data/nav.yml b/cms/data/nav.yml index 775984cd67..1330fd564b 100644 --- a/cms/data/nav.yml +++ b/cms/data/nav.yml @@ -29,6 +29,8 @@ entries: route: doaj.xml # ~~->XMLDocumentation:WebRoute~~ - label: Metadata help route: doaj.faq # ~~->FAQ:WebRoute~~ + - label: Preservation + route: doaj.preservation # ~~->Preservation:WebRoute~~ - id: about label: About footer: true diff --git a/cms/data/notifications.yml b/cms/data/notifications.yml index d4b9906822..71919c5ad4 100644 --- a/cms/data/notifications.yml +++ b/cms/data/notifications.yml @@ -40,15 +40,15 @@ application:publisher:accepted:notify: long: | Congratulations! The application submitted for **{application_title}** on {application_date} has been accepted for inclusion in DOAJ. - You may access the journal record from your Publisher dashboard: [{publisher_dashboard_url}]({publisher_dashboard_url}) using your DOAJ username and password. + You may access the journal record from your Publisher dashboard: [{publisher_dashboard_url}]({publisher_dashboard_url}) using your DOAJ account ID or email address, and password. - If there are changes or updates to the information about your journal at any time after it has been accepted, please submit an Update Request from your Publisher dashboard promptly. Please be aware that failure to do this may result in removal of your journal from DOAJ. + If there are changes or updates to the information about your journal at any time after it has been accepted, please [submit an Update Request](https://doaj.org/publisher/journal) from your Publisher dashboard promptly. Failure to do this promptly may result in the withdrawal of your journal from DOAJ. - How to submit an Update Request: [{faq_url}#how-do-i-update-the-information-about-my-journal]({faq_url}#how-do-i-update-the-information-about-my-journal) + [How to submit an Update Request](https://doaj.org/apply/publisher-responsibilities/#keeping-your-journal-records-up-to-date) To increase the visibility, impact, distribution and usage of your journal content, we encourage you to upload article metadata for this journal to DOAJ as soon as possible. - How to upload article metadata: [{faq_url}#how-do-i-upload-article-metadata]({faq_url}#how-do-i-upload-article-metadata) + [How to upload article metadata](https://doaj.org/docs/faq/#uploading-article-metadata) We are delighted to welcome this journal into DOAJ. Do not hesitate to contact us at [helpdesk@doaj.org](mailto:helpdesk@doaj.org) if you have any questions. short: diff --git a/cms/data/sponsors.yml b/cms/data/sponsors.yml index 2077eeb66e..38008a4621 100644 --- a/cms/data/sponsors.yml +++ b/cms/data/sponsors.yml @@ -25,6 +25,10 @@ url: https://www.ebsco.com/ logo: ebsco.svg +- name: Springer Nature Ltd + url: https://www.springernature.com/gp + logo: SN_stack_logo.png + - name: Frontiers url: https://www.frontiersin.org/ logo: frontiers.svg @@ -145,3 +149,7 @@ url: https://xueshu.libtools.com.cn/?tenant_id=1 logo: Beijing.jpg +- name: Association of Learned and Professional Society Publishers + url: https://www.alpsp.org/ + logo: ALPSP.png + diff --git a/cms/data/team.yml b/cms/data/team.yml index fc4dfed97e..5f7758cbc5 100644 --- a/cms/data/team.yml +++ b/cms/data/team.yml @@ -62,6 +62,7 @@ bio: "Ikhwan is a faculty member in the Industrial Engineering Department at Universitas Andalas, Indonesia. He is an enthusiastic photography hobbyist. He used to manage the university's library information system, including establishing the online public access catalogue (OPAC) and the repository servers. He is keenly interested in data science, primarily industrial and manufacturing systems research. He has been assisting Indonesian journal managers in their application to DOAJ. He joined DOAJ in 2018 as an ambassador, then became a volunteer editor and joined the DOAJ Team as Managing Editor in 2024." coi: 2022: https://drive.google.com/file/d/1bmVVvMAPToLQCHGOfNz0D80xi4WyZhwn/view?usp=sharing + 2024: https://drive.google.com/file/d/1grtW4MM0drzwG4hG66_Ib3RioYTvkk2p/view?usp=sharing - name: Ivonne Lujano role: Community Manager diff --git a/cms/data/volunteers.yml b/cms/data/volunteers.yml index 987d4a18db..7fbe11a1c8 100644 --- a/cms/data/volunteers.yml +++ b/cms/data/volunteers.yml @@ -18,21 +18,14 @@ ed: country: Australia language: Chinese, English featured: true - -- name: Ikhwan Arief - area: Science, Technology - year_since: - city: Padang, West Sumatra - country: Indonesia - language: Indonesian, English -- name: Juyeon Park - area: Librarianship +- name: Jiayi Xu + area: Academic Publishing year_since: - city: Seoul - country: Korea - language: Korean, English - + city: Chongqing + country: China + language: Chinese, English + - name: Lut Tamam area: Medicine year_since: @@ -47,6 +40,7 @@ ed: city: Surabaya country: Indonesia language: Indonesian, English + photo: "Ratodi.jpg" - name: Natalia Pamuła-Cieślak area: Humanities, Social Sciences @@ -81,6 +75,7 @@ ed: city: Tabriz country: Iran language: Persian, English, Azeri + photo: "Rasoul.jpg" - name: S.Suharto area: Humanities, Social Sciences @@ -100,6 +95,14 @@ ed: ## All associate editors ass_ed: +- name: Adhi Narayanan + area: Scientometric and Bibliometric Studies, Biotechnology + year_since: + city: Tiruchirappalli + country: India + language: Tamil, English + photo: "Dr. K.JPG" + - name: Alain Chaple area: Medical Sciences year_since: @@ -145,6 +148,7 @@ ass_ed: city: Madiun country: Indonesia language: Indonesian, English + photo: "Andista.jpg" - name: Andrea Imre area: Library Science, Music, General @@ -154,11 +158,12 @@ ass_ed: language: Hungarian, English - name: Andrea Marchitelli - area: Librarianship + area: Library Science year_since: - city: - country: Poland - language: Polish, English + city: Rome + country: Italy + language: Italian, English + photo: "Marchitelli.jpg" - name: Andronic Octavian area: Medicine @@ -166,6 +171,7 @@ ass_ed: city: Bucharest country: Romania language: Romanian, English + photo: "Octav.jpg" - name: Anna Sidorko area: Library Science @@ -218,6 +224,22 @@ ass_ed: language: Spanish, English photo: "CarlosAlegre.jpg" +- name: Carla Longo + area: Humanities + year_since: + city: St Andrews + country: Scotland + language: Italian, English, Latin, Ancient Greek + photo: "Longo.jpg" + +- name: Carla Marques + area: Information and library sciences + year_since: + city: Braga + country: Portugal + language: Information and library sciences + photo: "Carla.jpg" + - name: Cezary Borkowicz area: Librarianship year_since: @@ -240,7 +262,7 @@ ass_ed: city: Bengkulu country: Indonesia language: Indonesian, English - photo: + photo: "Danner..png" - name: Daria Chrześcijańska area: Computer Science, Social Sciences @@ -272,6 +294,7 @@ ass_ed: city: Samarinda country: Indonesia language: Indonesian, English + photo: "DEDI.jpg" - name: Dessy Harisanty area: Library and Information Science @@ -326,13 +349,6 @@ ass_ed: language: English photo: "Emrah Kaya.jpg" -- name: Francesca Soldati - area: Science - year_since: - city: Lincoln - country: United Kingdom - language: Italian, English, Spanish - - name: Francesco Cavinato area: Humanities, Social Sciences year_since: @@ -362,7 +378,6 @@ ass_ed: country: Republic of Korea language: Korean, English - - name: Iryna Kuchma area: Humanities, Social Sciences year_since: @@ -376,6 +391,7 @@ ass_ed: city: Valencia country: Spain language: Spanish, English + photo: "Margalejo.jpeg" - name: Jamila Jaber area: Library and Information Science @@ -391,13 +407,6 @@ ass_ed: country: language: English -- name: Jiayi Xu - area: Publishing - year_since: - city: Chongqing - country: China - language: Chinese,English - - name: Jinjin Liu area: Social Sciences, Medicine year_since: @@ -405,13 +414,12 @@ ass_ed: country: China language: English -- name: João Sousa - area: Social Sciences +- name: João Pedro Oliveira + area: Health Sciences year_since: - city: - country: Angola + city: Porto + country: Portugal language: Portuguese, English - photo: "sousa.JPG" - name: JooYeun Son area: @@ -420,13 +428,6 @@ ass_ed: country: language: Korean, English -- name: José Darío Martínez Ezquerro - area: Medicine - year_since: - city: - country: Mexico - language: Spanish, English - - name: Juliana Soares Lima area: Librarianship and information science year_since: @@ -435,11 +436,19 @@ ass_ed: language: Portuguese, English photo: "jlima.jpg" -- name: JuYeon Park - area: +- name: Julio Zetter + area: Library Science year_since: - city: - country: + city: Mexico City + country: Mexico + language: Spanish, English + photo: "Julio.jpg" + +- name: Juyeon Park + area: Librarianship + year_since: + city: Seoul + country: Korea language: Korean, English - name: Kadri Kıran @@ -473,14 +482,7 @@ ass_ed: city: country: Indonesia language: Indonesian, English - photo: - -- name: Kimberly Mears - area: Medicine - year_since: - city: Charlottetown - country: Canada - language: English + photo: "Khoirul.jpg" - name: Kristen Totleben area: Librarianship, Humanities, Social Sciences @@ -490,12 +492,28 @@ ass_ed: language: English, Spanish photo: "Kristen.jpg" +- name: Lee Yang Díaz Chieng + area: Information Sciences + year_since: + city: Guantánamo + country: Cuba + language: Spanish and English + photo: "Lee.jpg" + - name: Lena Lönngren area: year_since: city: country: Finland language: Finnish, English + +- name: Leila Yang + area: Scholarly Publishing + year_since: + city: Beijing + country: China + language: Chinese, English + photo: "Yang.jpg" - name: Liana Penabad Camacho area: Social Sciences @@ -503,6 +521,7 @@ ass_ed: city: San José country: Costa Rica language: Spanish, English + photo: "Penabad.png" - name: Lina Marcela Blandón García area: Science @@ -511,6 +530,14 @@ ass_ed: country: Colombia language: Spanish, English +- name: Liping Yang + area: Material Engineering, Library Studies + year_since: + city: Suzhou + country: China + language: Chinese, English + photo: "Liping.jpg" + - name: Lorri Peters area: Science year_since: @@ -518,20 +545,13 @@ ass_ed: country: USA language: English -- name: Lucas Helal - area: Public Health - year_since: - city: Criciúma - country: Brazil - language: Portuguese, Spanish, English - photo: "lucashelal.jpg" - - name: Marcau Flavius-Cristian area: Social Sciences year_since: city: Târgu jiu country: Romania language: Romanian, English + photo: "Flavius.jpg" - name: Marco Tullney area: Social Sciences, Technology @@ -553,14 +573,6 @@ ass_ed: city: V.N Gaia country: Portugal language: Portuguese, Spanish, English - -- name: Mariano Hernán Corujo - area: Librarianship - year_since: - city: Buenos Aires - country: Argentina - language: Spanish, English - photo: "MarianoCorujo.jpeg" - name: Marie-Eve Dugas area: Library Science, Social Sciences @@ -576,6 +588,14 @@ ass_ed: country: language: Spanish, English +- name: Martyna Mirecka + area: History, Political Science + year_since: + city: Łupice + country: Poland + language: Polish, English + photo: "Martyna.JPG" + - name: Melih Sever area: Social Sciences year_since: @@ -583,14 +603,6 @@ ass_ed: country: Türkiye language: English -- name: Miguel-Ángel Vera-Baceta - area: Social Sciences - year_since: - city: Murcia - country: Spain - language: Spanish, English - photo: "MiguelAVera.jpg" - - name: Milagro Castro area: Social Sciences year_since: @@ -675,6 +687,7 @@ ass_ed: city: Donostia country: Spain language: Euskera, Spanish, English + photo: "Agirre.JPG" - name: Paula Carina de Araújo area: Library and Information Science @@ -707,13 +720,6 @@ ass_ed: country: Indonesia language: Indonesian, English -- name: Raúl Sampieri - area: Science - year_since: - city: - country: Mexico - language: Spanish, English - - name: Remedios Melero (Reme) area: Social Sciences year_since: @@ -757,13 +763,12 @@ ass_ed: country: Italy language: Italian, English, German -- name: Sofia Fagiolo - area: Librarianship +- name: Shiying Li + area: Forensic Science year_since: - city: Rome - country: Italy - language: Italian, English - photo: "sofiaf.jpg" + city: Shanghai + country: China + language: Chinese - name: Soon Kim area: @@ -787,6 +792,14 @@ ass_ed: language: French photo: "sogoba1.jpg" +- name: Susana Costa + area: Library and Information Sciences + year_since: + city: Braga + country: Portugal + language: Portuguese, English + photo: "Costa.jpeg" + - name: Swasti Maharani area: Mathematics year_since: @@ -816,6 +829,14 @@ ass_ed: country: India language: Kannada, English photo: "Vasanth.jpg" + +- name: Wileidys Artigas + area: Library Science + year_since: + city: Plano, Texas + country: USA + language: Spanish, English and Portuguese + photo: "Wileidys.jpg" - name: Yalçın Tükel area: Sports recreation diff --git a/cms/pages/apply/copyright-and-licensing.md b/cms/pages/apply/copyright-and-licensing.md index 2f9b41443a..15c5e29bf7 100644 --- a/cms/pages/apply/copyright-and-licensing.md +++ b/cms/pages/apply/copyright-and-licensing.md @@ -49,3 +49,11 @@ But the license applies to the readers **and** the author when: ## Further reading Further reading and more examples are [available as a downloadable presentation](https://drive.google.com/drive/folders/190BgMV0ImGk-gUpHu5ai_R-uvO8NDAB8?usp=sharing). + +## In other languages + +[French](https://www.erudit.org/public/documents/licencedroitsauteursDOAJ.pdf) - hosted by Érudit + +## Version history + +This is Version 1 of our Licensing and copyright page. diff --git a/cms/pages/apply/guide.md b/cms/pages/apply/guide.md index 46ac31918c..854c2f5bf7 100644 --- a/cms/pages/apply/guide.md +++ b/cms/pages/apply/guide.md @@ -223,7 +223,7 @@ Our criteria are available in: - [Chinese](https://zenodo.org/record/4633341) - [Danish](https://pro.kb.dk/danske-open-access-tidsskrifter-og-directory-open-access-journals/basisbetingelser-indeksering) - [Finnish](https://docs.google.com/document/d/1BLuaFerSw0G4L2GCVcGeu3rB7SXT7fWL0px26ME9jo0/edit?usp=sharing) -- [French (Canadian)](https://drive.google.com/drive/folders/1-9FraAimhA9Ks64tvKE_W4wsQ8LNob9a?usp=sharing) +- [French](https://www.erudit.org/public/documents/guidecandidatureDOAJ.pdf) - hosted by Érudit - [German](https://bibliothek.thws.de/leitfaden-fuer-die-zeitschriftenregistrierung-bei-doaj/) - [Japanese](https://drive.google.com/file/d/1MDRlcc7SJnv8yOlZ1aCqbivXOZevxH4a/view?usp=sharing) - [Lithuanian](https://drive.google.com/file/d/1f7YXn6cXGXhDH9AbPJyOiM7_tHVRMe4j/view?usp=sharing) @@ -234,7 +234,7 @@ Our criteria are available in: --- -## Change log +## Version history This is Version 2.2 of our Guide to applying. diff --git a/cms/pages/apply/publisher-responsibilities.md b/cms/pages/apply/publisher-responsibilities.md index 75d54a0a5f..007616c8d7 100644 --- a/cms/pages/apply/publisher-responsibilities.md +++ b/cms/pages/apply/publisher-responsibilities.md @@ -75,7 +75,7 @@ You can access your dashboard and account settings by logging into your account. On [your dashboard](/publisher/), you can - See [a list of your journals](/publisher/journal) indexed in DOAJ -- [Submit an update request](/apply/guide/#updating-your-journal-record) when journal details change +- [Submit an update request](/publisher-responsibilities/#keeping-your-journal-records-up-to-date) when journal details change - [Upload](/publisher/uploadfile) or [enter](/publisher/metadata) article metadata for your journals - Download the Seal logo (only for journals awarded the DOAJ Seal). - Upload your full-text content (only for journals preserved via JASPER). @@ -163,7 +163,7 @@ When we withdraw your journal, we will send you an email. Remind yourself of the - We have to withdraw your journal from DOAJ if it no longer adheres to our criteria or to publishing best practices. - We have to withdraw the journal if it is inactive (ceased publishing) or the website is unavailable. -You can also check our [list of withdrawn journals](https://docs.google.com/spreadsheets/d/183mRBRqs2jOyP0qZWXN8dUd02D4vL0Mov_kgYF8HORM/edit#gid=1650882189) and ask for more information by [contacting our Help Desk](mailto:helpdesk@doaj.org). +You can also check our [log of withdrawn journals](https://docs.google.com/spreadsheets/d/1Kv3MbgFSgtSDnEGkA2JacrSjunRu0umHeZCtcMeqO5E/edit#gid=2104690845&range=A1) and ask for more information by [contacting our Help Desk](mailto:helpdesk@doaj.org). ### Journal is not listed on my dashboard @@ -172,3 +172,11 @@ You can also check our [list of withdrawn journals](https://docs.google.com/spre ### Unknown journal on my dashboard - [Contact our Help Desk](mailto:helpdesk@doaj.org) with the journal title and ISSN; they will help you. + +## In other languages + +[French](https://www.erudit.org/public/documents/infoediteursDOAJ.pdf) - hosted by Érudit + +## Version history + +This is Version 1 of our Publisher information page. diff --git a/cms/pages/apply/seal.md b/cms/pages/apply/seal.md index 7fc7f4d414..1a1e6984f3 100644 --- a/cms/pages/apply/seal.md +++ b/cms/pages/apply/seal.md @@ -47,3 +47,11 @@ All seven criteria must be met for a journal to be awarded the Seal. Failure to - Creative Commons licensing information must be displayed in all full-text article formats. 7. Copyright and publishing rights - Authors must retain unrestricted copyright and all publishing rights when publishing under any license permitted by the journal. + +## In other languages + +[French](https://www.erudit.org/public/documents/sceauDOAJ.pdf) - hosted by Érudit + +## Version history + +This is Version 1 of our Seal criteria. diff --git a/cms/pages/apply/transparency.md b/cms/pages/apply/transparency.md index 7cb39e0a0c..ec05c1d83a 100644 --- a/cms/pages/apply/transparency.md +++ b/cms/pages/apply/transparency.md @@ -171,7 +171,7 @@ Any direct marketing activities, including solicitation of manuscripts, that are ## Version history -- This is Version 4.0 of the Principles of Transparency and Best Practice in Scholarly Publishing +- This is Version 4.0 - September 2022 - [Version 3.0](https://docs.google.com/document/d/1wtnA5zj02Hn3XY_SUWvs3U22DC9MaaNVrdiCHYXlrGE/edit?usp=sharing) - January 2018 - [Version 2.0](https://oaspa.org/principles-of-transparency-and-best-practice-in-scholarly-publishing-2/?highlight=principles) - June 2015 (on the OASPA website) - [Version 1.0](https://oaspa.org/principles-of-transparency-and-best-practice-in-scholarly-publishing/?highlight=principles) - December 2013 (on the OASPA website) @@ -203,6 +203,7 @@ WAME is a global nonprofit voluntary association of editors of peer-reviewed med - [Bengali](https://docs.google.com/document/d/1hsCynqvYbnaUwnu7VxlEPNKn076mz1KrRUdQ4vefYX4/edit?usp=sharing) - Google document - [Chinese](https://zenodo.org/records/10401800) - PDF, Zenodo +- [French](https://www.erudit.org/public/documents/transparenceDOAJ.pdf) - hosted by Érudit - [Korean](https://drive.google.com/file/d/1Uyh5uZR1vbkPwcIvN1h7PPlHVwu0fMHA/view?usp=sharing) - PDF - [Portuguese](https://docs.google.com/document/d/155dAHllL2KhPhzTsR3UhbMEASYjkxP157fAtiZ1jw2w/edit?usp=sharing) - Google document - [Serbian](https://www.ceon.rs/index.php?option=com_content&view=article&id=654:transparentnost-i-najbolja-praksa&catid=94&lang=sr&Itemid=578) (hosted at ceon.rs) diff --git a/cms/pages/docs/faq.md b/cms/pages/docs/faq.md index 6621c19bf2..c2fe321bc4 100644 --- a/cms/pages/docs/faq.md +++ b/cms/pages/docs/faq.md @@ -10,10 +10,10 @@ featuremap: ~~FAQ:Fragment~~ After your journal is indexed in DOAJ and you start to upload article metadata to us, we generate journal and article metadata. We make these publicly and freely available via different methods: -- Our [Atom feed](https://staticdoaj.cottagelabs.com/feed) -- Our [OAI-PMH service](https://staticdoaj.cottagelabs.com/docs/oai-pmh/) -- [A journal CSV](https://staticdoaj.cottagelabs.com/csv) file (updates every 60 minutes) -- Our [API](https://staticdoaj.cottagelabs.com/docs/api/) +- Our [Atom feed](https://doaj.org/feed) +- Our [OAI-PMH service](https://doaj.org/docs/oai-pmh/) +- [A journal CSV](https://doaj.org/csv) file (updates every 60 minutes) +- Our [API](https://doaj.org/docs/api/) - On our website Our metadata is collected and incorporated into commercial discovery systems, library discovery portals and search engines around the world. Here are some of them: @@ -100,6 +100,20 @@ Choose how you want to upload article metadata to us. - **Documentation** No - **Troubleshooting**: you must be careful to enter the Print ISSN and Electronic ISSN in the right field. +### A note about updating articles + +Sometimes article metadata needs to be updated. We use the Full Text URL and DOI, if present, to identify and match articles. If you are updating article metadata with details about authors, affiliations, year, volume, issue, etc or adding a DOI to an article that doesn't have one in DOAJ, you will be able to send us the updated metadata, using one of the methods above, without any problem. + +However, if you need to update the Full Text URL or DOI of articles that already have that metadata present in DOAJ, you will need to contact us first. Submitting new FUll Text URLs or DOIs will cause duplicate articles to be created. Our preferenc here is that we delete the existing article records first. + +If you do need us to delete article metadata, please contact Help Desk with the following details: + +- journal title +- ISSN(s) +- years to be deleted (we can only delete whole years) or if we should delete all article for a journal + +We will confirm the number of articles to delete with you. Deleting articles is instantaneous and cannot be reversed. + ## Help with metadata uploads ### My authors have multiple affiliations @@ -144,11 +158,12 @@ If you received a spreadsheet from us, please complete it as soon as possible. T Before you send us the file, you must do two things: -1. Convert the spreadsheet to a CSV. To do this, you will need to first delete the instructions tab and then Save as CSV. +1. Convert the spreadsheet to a CSV. To do this, you will need to first delete the instructions tab and then Save as CSV. (Save in the Unicode UTF-8 format.) 2. [Validate it](/publisher/journal-csv). Here are some tips on how to ensure that your CSV file will pass validation: +- make sure CSV is in UTF-8 format - don't change an ISSN or Title of a journal. To do this, contact [Help Desk](mailto:helpdesk@doaj.org). - don't add a new journal to the file. To do this, [submit a new application](/apply/). - don't change the title of a column @@ -174,4 +189,11 @@ The following warnings may be seen after validating your CSV: | You may not change _question_. Please revert it to match what was sent to you in the spreadsheet. | During editing, the question has been changed. The question must be exactly as it is in the spreadsheet sent to you. Please change it. | | We couldn't understand the information in _question_ | | The information in the cell doesn't match the formatting requirements. Check the Instructions tab in the spreadsheet sent to you. | -From time to time, other validation errors might be seen if one of the cells contains completely incorrect information. For example, the cell should contain a URL but it contains text. These error messages are self-explanatory,, but contact Help Desk if you require help. +From time to time, other validation errors might be seen if one of the cells contains completely incorrect information. For example, the cell should contain a URL but it contains text. These error messages should be easy to understand but contact the Help Desk if you have questions. + +## Version history + +This is Version 2 of our Metadata help page. + +*Version 2.0 (December 2023 - added the entire 'Using a spreadsheet to update your journal metadata' section)*
+Version 1.0 (November 2023 - created this whole page with new content)
diff --git a/cms/pages/support/supporters.md b/cms/pages/support/supporters.md index 6cf98e6b0f..eed80a4508 100644 --- a/cms/pages/support/supporters.md +++ b/cms/pages/support/supporters.md @@ -15,4 +15,4 @@ Check [our Institutions and libraries support page](/support/) for pricing and b ## Current supporters - + diff --git a/cms/sass/base/_palette.scss b/cms/sass/base/_palette.scss index f33e96f69f..c180436f5c 100644 --- a/cms/sass/base/_palette.scss +++ b/cms/sass/base/_palette.scss @@ -3,6 +3,7 @@ // Greyscale $warm-black: #282624; $dark-grey: #5C5956; +$mid-grey: #A9A7A5; $light-grey: #F6F4F4; $white: #FFF; diff --git a/cms/sass/components/_form.scss b/cms/sass/components/_form.scss index 81eb330e81..8d3e93ee06 100644 --- a/cms/sass/components/_form.scss +++ b/cms/sass/components/_form.scss @@ -11,6 +11,14 @@ } } +.form__legend { + margin-bottom: $spacing-03; + font-weight: 700; + line-height: 1; + @include typescale-01; + @include font-serif; +} + .form__header { margin-bottom: 0; padding-bottom: $spacing-03; diff --git a/cms/sass/components/_notifications.scss b/cms/sass/components/_notifications.scss index 35edadb64a..a7975845c4 100644 --- a/cms/sass/components/_notifications.scss +++ b/cms/sass/components/_notifications.scss @@ -1,6 +1,6 @@ .notifications { overflow-y: scroll; - height: 300px; + height: fit-content; width: 300px; @include typescale-06; } diff --git a/cms/sass/layout/_editorial-panel.scss b/cms/sass/layout/_editorial-panel.scss index a08cb98ca3..0f2f818dd0 100644 --- a/cms/sass/layout/_editorial-panel.scss +++ b/cms/sass/layout/_editorial-panel.scss @@ -16,8 +16,14 @@ position: sticky; top: 100px; + &__wrapper { + display: flex; + flex-direction: column; + max-height: calc(100vh - 75px - 1.5rem - 25px); //(full viewport)-(nav height)-(top padding)-(bottom padding) + } + &__content { - max-height: 70vh; + flex: 1; overflow-y: scroll; select, textarea, input { diff --git a/cms/sass/main.scss b/cms/sass/main.scss index 661f2a4ae6..0f72c8eac8 100644 --- a/cms/sass/main.scss +++ b/cms/sass/main.scss @@ -14,7 +14,7 @@ "base/general", "base/grid", "base/highlight", - + "vendors/swagger", "layout/editorial-panel", @@ -71,5 +71,6 @@ "pages/uploadmetadata", "themes/dashboard", + "themes/editorial-form", "themes/timeline" ; diff --git a/cms/sass/themes/_dashboard.scss b/cms/sass/themes/_dashboard.scss index 50bd3e8986..5d68af6374 100644 --- a/cms/sass/themes/_dashboard.scss +++ b/cms/sass/themes/_dashboard.scss @@ -58,6 +58,7 @@ top: 75px; bottom: 0; background-color: $light-grey; + border-right: 1px dotted $mid-grey; overflow-y: scroll; } } @@ -171,4 +172,136 @@ padding: 40px; } } + + // Application form styles — overrides + .form { + padding: $spacing-04 0; + } + + .form--compact { + h1 { + margin: 0 (-$spacing-04) $spacing-04 (-$spacing-04); + padding: $spacing-04; + border-radius: $spacing-02 $spacing-02 0 0; + background: $warm-black; + color: $white; + + + .form__header { + margin-top: -$spacing-04; + } + + + .alert { + margin: 0 0 $spacing-04 0; + background-color: $light-grey; + border-radius: $spacing-02; + border-color: $mid-grey; + color: $warm-black; + } + + + .form__question { + padding-top: 0; + border: 0; + } + } + + .form__header { + margin: 0 (-$spacing-04); + padding: $spacing-03 $spacing-04; + width: auto; + border: 0; + border-top: 1px dotted $mid-grey; + background: $light-grey; + } + + fieldset { + margin: 0 0 $spacing-04 0; + padding: 0 $spacing-04; + border-radius: $spacing-02; + background: $white; + } + + .form__question { + padding: $spacing-04; + margin: 0; + margin-left: -$spacing-04; + margin-right: -$spacing-04; + border-top: 1px dotted $mid-grey; + + input, select, textarea, .select2-container { + margin-bottom: $spacing-03; + width: 100% !important; + border-color: $mid-grey; + } + + .removable-fields li { + display: flex; + justify-content: center; + } + + .remove_field__button { + margin-bottom: $spacing-03; + white-space: nowrap; + } + } + + .form__subquestion { + border-left-width: $spacing-01; + } + + .form__short-help { + margin-bottom: $spacing-02; + font-size: smaller; + } + + .icon-container { + display: inline-flex; + margin-bottom: $spacing-02; + padding: $spacing-01; + line-height: 1; + border-radius: 50%; + color: $white; + } + + .icon-container--unable_to_access { + background: $grapefruit; + } + + .icon-container--not_found, + .icon-container--not_validated, + .icon-container--missing, + .icon-container--outdated { + background: $sanguine; + } + + .icon-container--fully_validated, + .icon-container--present { + background: $mid-green; + } + + .parsley-errors-list { + padding: $spacing-03; + border-radius: $spacing-01; + background: rgba($grapefruit, .25); + font-weight: bold; + + p { + margin: 0; + } + } + + .formulaic-annotation-pissn-list, + .formulaic-annotation-eissn-list, + .formulaic-annotation-preservation_service-list { + margin: 0; + padding: $spacing-03; + border-radius: $spacing-02; + border: 1px dotted $mid-grey; + background: $light-grey; + } + + .formulaic-clickableurl-visit { + margin: 0 0 0 $spacing-03; + white-space: nowrap; + } + } } diff --git a/cms/sass/themes/_editorial-form.scss b/cms/sass/themes/_editorial-form.scss new file mode 100644 index 0000000000..3e2ceb4c45 --- /dev/null +++ b/cms/sass/themes/_editorial-form.scss @@ -0,0 +1,197 @@ +// Application form review styles — overrides +// Managing editor, editor, associate editor views +#maned_form, +#ed_form, +#assed_form { + padding: $spacing-04 0; + background: rgba($light-grey, .5); + + details { + cursor: pointer; + } + + fieldset { + margin: 0 0 $spacing-04 0; + padding: 0 $spacing-04; + border-radius: $spacing-02; + background: $white; + @include box-shadow; + } + + .page-content { + padding: 0; + } + + .form__legend { + margin: 0; + width: 100%; + @include typescale-06; + + div { + margin: 0 (-$spacing-04) $spacing-04 (-$spacing-04); + padding: $spacing-02; + border-radius: $spacing-02 $spacing-02 0 0; + background: $dark-grey; + color: $white; + text-align: center; + } + + + .form__header, + + details { + margin-top: -$spacing-04; + } + + + .alert { + margin: 0 0 $spacing-04 0; + background-color: $light-grey; + border-radius: $spacing-02; + border-color: $mid-grey; + color: $warm-black; + } + + + .form__question { + padding-top: 0; + border: 0; + } + } + + .form__header { + margin: 0 (-$spacing-04); + padding: $spacing-03; + width: auto; + border: 0; + border-top: 1px dotted $mid-grey; + background: $light-grey; + } + + .form__question { + padding: $spacing-03; + margin: 0; + margin-left: -$spacing-04; + margin-right: -$spacing-04; + border-top: 1px dotted $mid-grey; + + input, select, textarea, .select2-container { + margin-bottom: $spacing-02; + width: 100% !important; + border-color: $mid-grey; + } + + .removable-fields li { + display: flex; + justify-content: center; + } + + .remove_field__button { + margin-bottom: $spacing-03; + white-space: nowrap; + } + + &.editor_group__container, + &.editor__container { + border-top: 0; + } + } + + .form__subquestion { + border-left-width: $spacing-01; + } + + .form__short-help { + margin-bottom: $spacing-02; + font-size: smaller; + } + + .icon-container { + display: inline-flex; + margin-right: $spacing-02; + padding: $spacing-01; + line-height: 1; + border-radius: 50%; + color: $white; + } + + .icon-container--success { + background: $mid-green; + } + + .icon-container--error { + background: $sanguine; + } + + .icon-container--warn { + background: $grapefruit; + } + + .icon-container--info { + background: $mid-grey; + } + + //.icon-container--unable_to_access { + // background: $grapefruit; + //} + // + //.icon-container--not_found, + //.icon-container--not_validated, + //.icon-container--missing, + //.icon-container--outdated { + // background: $sanguine; + //} + // + //.icon-container--fully_validated, + //.icon-container--present { + // background: $mid-green; + //} + + .parsley-errors-list { + padding: $spacing-03; + border-radius: $spacing-01; + background: rgba($grapefruit, .25); + font-weight: bold; + + p { + margin: 0; + } + } + + .formulaic-annotation-pissn-list, + .formulaic-annotation-eissn-list, + .formulaic-annotation-preservation_service-list { + margin: 0; + padding: $spacing-03; + border-radius: $spacing-02; + border: 1px dotted $mid-grey; + background: $light-grey; + + li { + display: flex; + align-items: center; + justify-content: center; + line-height: 1; + + + li { + margin-top: $spacing-02; + } + } + + button { // dismiss button + margin: 0; + margin-left: auto; + } + } + + .formulaic-clickableurl-visit { + margin: 0 0 0 $spacing-03; + white-space: nowrap; + } + + [class*="-annotations"] { + + [class*="fullcontents"] { + margin: $spacing-03 0 0 0; + } + } + + [class*="fullcontents-contents"] { + margin: 0; + } +} \ No newline at end of file diff --git a/cms/tours/admin_journal_autochecks.yml b/cms/tours/admin_journal_autochecks.yml new file mode 100644 index 0000000000..7c6e575489 --- /dev/null +++ b/cms/tours/admin_journal_autochecks.yml @@ -0,0 +1,14 @@ +steps: + - selector: '.autochecks-manager-toggle' + title: Autochecks are available on this Journal + content: | +
+

This journal has had a number of checks carried out on its data, to aid in your review. To see these checks alongside + the questions in the form, click this button to show them.

+

For now these are quite basic, covering the following fields:

+
    +
  1. Print and Online ISSNs: these are checked to see if they are registered and valid in issn.org
    2. +
  2. Archiving policy: these are checked to see if the journal is being archived in the selected services, and how current the data is.
    3. +
+

In time, more checks will become available.

+
\ No newline at end of file diff --git a/data_import_settings/dev_extras.json b/data_import_settings/dev_extras.json index b2bd93258d..8525b6e9d2 100644 --- a/data_import_settings/dev_extras.json +++ b/data_import_settings/dev_extras.json @@ -1,24 +1,24 @@ { "elastic_search_host" : null, "elastic_search_db" : null, - "confirm" : true, + "confirm" : false, "max_content_length" : 40000000, "types" : { "account" : {"import" : true, "limit" : -1}, "application" : {"import" : true, "limit" : -1}, - "article" : {"import" : true, "limit" : 100000}, "background_job" : {"import" : true, "limit" : 10000}, - "news" : {"import" : true, "limit" : -1}, - "notification" : {"import" : true, "limit" : -1}, - "preserve" : {"import" : true, "limit" : -1}, + "cache" : {"import" : true, "limit" : -1}, + "draft_application" : {"import" : true, "limit" : 1000}, "editor_group" : {"import" : true, "limit" : -1}, + "harvester_state" : {"import" : true, "limit" : -1}, "journal" : {"import" : true, "limit" : -1}, - - "cache" : {"import" : false, "limit" : -1}, - "harvester_state" : {"import" : false, "limit" : -1}, - "lcc" : {"import" : false, "limit" : -1}, - "lock" : {"import" : false, "limit" : -1}, - "provenance" : {"import" : false, "limit" : -1}, - "upload" : {"import" : false, "limit" : -1} + "lcc" : {"import" : true, "limit" : -1}, + "lock" : {"import" : true, "limit" : -1}, + "news" : {"import" : true, "limit" : -1}, + "notification" : {"import" : true, "limit" : 1000}, + "preserve" : {"import" : true, "limit" : -1}, + "provenance" : {"import" : true, "limit" : 10000}, + "upload" : {"import" : true, "limit" : 10000}, + "article" : {"import" : true, "limit" : 100000} } } \ No newline at end of file diff --git a/doajtest/README.md b/doajtest/README.md index 4f5b85873d..fb7b0415d9 100644 --- a/doajtest/README.md +++ b/doajtest/README.md @@ -33,6 +33,12 @@ Following are the guide to write unit test for parallelised: ---------------------------------------- +## Combinatrix and paramaterised tests + +Find the test matrices here: https://drive.google.com/drive/folders/1EFKBbBCay983wzSUgD6F3Bj_V8c5ix54 + +TODO, full guide... + ## Testbook See: Testbook section from ../docs/README.md diff --git a/doajtest/fixtures/__init__.py b/doajtest/fixtures/__init__.py index 75d594d773..156fe944e5 100644 --- a/doajtest/fixtures/__init__.py +++ b/doajtest/fixtures/__init__.py @@ -6,3 +6,4 @@ from .bibjson import BibJSONFixtureFactory from .provenance import ProvenanceFixtureFactory from .background import BackgroundFixtureFactory +from doajtest.fixtures.issn_org import IssnOrgFixtureFactory diff --git a/doajtest/fixtures/issn_org.py b/doajtest/fixtures/issn_org.py new file mode 100644 index 0000000000..72addfbed5 --- /dev/null +++ b/doajtest/fixtures/issn_org.py @@ -0,0 +1,9 @@ +from copy import deepcopy +from portality.lib import paths + +class IssnOrgFixtureFactory(object): + @classmethod + def web_page_body(cls): + source = paths.rel2abs(__file__, "../unit/resources/issn_org_web_page.html") + with open(source) as f: + return f.read() \ No newline at end of file diff --git a/doajtest/fixtures/resources.py b/doajtest/fixtures/resources.py new file mode 100644 index 0000000000..69bb9e9364 --- /dev/null +++ b/doajtest/fixtures/resources.py @@ -0,0 +1,123 @@ +from copy import deepcopy +from datetime import datetime + +from portality.autocheck.resources.issn_org import ISSNOrgData + + +class ResourcesFixtureFactory(object): + @classmethod + def issn_org(cls, issn=None, version=None, archive_components=None): + record = deepcopy(ISSN_ORG) + + if issn is not None: + record["@id"] = "https://portal.issn.org/resource/ISSN/" + issn + record["exampleOfWork"]["@id"] = record["@id"] + record["exampleOfWork"]["workExample"][0]["@id"] = record["@id"] + record["issn"] = issn + record["identifier"][0]["value"] = issn + record["identifier"][1]["value"] = issn + record["mainEntityOfPage"]["@id"] = record["@id"] + "#Record" + record["mainEntityOfPage"]["mainEntity"] = record["@id"] + + if version is not None: + record["mainEntityOfPage"]["version"] = version + + if archive_components is not None: + record["subjectOf"] = [] + for service, in_time in archive_components.items(): + ac_base = deepcopy(SUBJECT_OF) + service_url = ID_MAP.get(service) + ac_base["holdingArchive"]["@id"] = service_url + ac_base["holdingArchive"]["name"] = service + if in_time: + now = datetime.utcnow() + last = now.year - 1 + first = now.year - 3 + ac_base["temporalCoverage"] = "{x}/{y}".format(x=first, y=last) + else: + now = datetime.utcnow() + last = now.year - 4 + first = now.year - 6 + ac_base["temporalCoverage"] = "{x}/{y}".format(x=first, y=last) + + record["subjectOf"].append(ac_base) + + return ISSNOrgData(record) + + +ID_MAP = { + "CLOCKSS": "http://issn.org/organization/keepers#clockss", + "LOCKSS": "http://issn.org/organization/keepers#lockss", + "Internet Archive": "http://issn.org/organization/keepers#internetarchive", + "PKP PN": "http://issn.org/organization/keepers#pkppln", + "Portico": "http://issn.org/organization/keepers#portico" +} + +SUBJECT_OF = { + "@type": "ArchiveComponent", + "creativeWorkStatus": "Preserved", + "description": "1 to 3", + "holdingArchive": { + "@type": "ArchiveOrganization", + "@id": "http://issn.org/organization/keepers#clockss", + "name": "CLOCKSS Archive" + }, + "abstract": "1 to 3", + "dateModified": "2023-06-19", + "temporalCoverage": "" +} + +ISSN_ORG = { + "@context": "http://schema.org/", + "@id": "https://portal.issn.org/resource/ISSN/1234-1231", + "@type": "Periodical", + "exampleOfWork": { + "@id": "https://portal.issn.org/resource/ISSN-L/1234-1231", + "@type": "CreativeWork", + "workExample": [ + { + "@id": "https://portal.issn.org/resource/ISSN/1234-1231", + "name": "Wiadomości Unii Spółdzielców Mieszkaniowych" + } + ] + }, + "issn": "1234-1231", + "identifier": [ + { + "@type": "PropertyValue", + "name": "ISSN", + "value": "1234-1231", + "description": "Valid" + }, + { + "@type": "PropertyValue", + "name": "ISSN-L", + "value": "1234-1231", + "description": "Valid" + } + ], + "name": "Wiadomości Unii Spółdzielców Mieszkaniowych", + "alternateName": "Wiadomości Unii Spółdzielców Mieszkaniowych.", + "publication": { + "@id": "https://portal.issn.org/resource/ISSN/1234-1231#ReferencePublicationEvent", + "@type": "PublicationEvent", + "location": { + "@id": "https://www.iso.org/obp/ui/#iso:code:3166:PL", + "@type": "Country", + "name": "Poland" + } + }, + "mainEntityOfPage": { + "@id": "https://portal.issn.org/resource/ISSN/1234-1231#Record", + "@type": "CreativeWork", + "dateModified": "05/01/2023", + "mainEntity": "https://portal.issn.org/resource/ISSN/1234-1231", + "sourceOrganization": { + "@id": "https://www.issn.org/organization/ISSNCenter#57", + "@type": "Organization", + "name": "ISSN National Centre for Poland" + }, + "version": "Register" + }, + "material": "Print" +} \ No newline at end of file diff --git a/doajtest/fixtures/urls.py b/doajtest/fixtures/urls.py index 0fa1f98302..df3ceb8949 100644 --- a/doajtest/fixtures/urls.py +++ b/doajtest/fixtures/urls.py @@ -4,7 +4,10 @@ "https://www.cosmos.com#galaxy", "https://www.cosmos.com/galaxy", "https://www.cosmos.com/galaxy#peanut", - "http://ftp.example.com/file%20name.txt" + "http://ftp.example.com/file%20name.txt", + "https://revistalogos.policia.edu.co:8443/index.php/rlct/about", + "https://revistalogos.policia.edu.co:65535/index.php/rlct/about", + "https://revistalogos.policia.edu.co:0/index.php/rlct/about" ] INVALID_URL_LISTS = [ @@ -12,7 +15,9 @@ "nonexistent.com", "https://www.doaj.org and https://www.reddit.com", "http://www.doaj.org and www.doaj.org", -"http://www.doaj.org, www.doaj.org", -"http://www.doaj.org, https://www.doaj.org", -"http://ftp.example.com/file name.txt" -] \ No newline at end of file + "http://www.doaj.org, www.doaj.org", + "http://www.doaj.org, https://www.doaj.org", + "http://ftp.example.com/file name.txt", + "https://revistalogos.policia.edu.co:65536/index.php/rlct/about", + "https://revistalogos.policia.edu.co:655350/index.php/rlct/about" +] diff --git a/doajtest/fixtures/v2/journals.py b/doajtest/fixtures/v2/journals.py index 068b14aebd..3b3d1123aa 100644 --- a/doajtest/fixtures/v2/journals.py +++ b/doajtest/fixtures/v2/journals.py @@ -118,8 +118,8 @@ def question_answers(): "Languages in which the journal accepts manuscripts", "Publisher", "Country of publisher", - "Society or institution", - "Country of society or institution", + "Other organisation", + "Country of other organisation", "Journal license", "License attributes", "URL for license terms", diff --git a/doajtest/helpers.py b/doajtest/helpers.py index 60c8dd0e85..a8f33d52d5 100644 --- a/doajtest/helpers.py +++ b/doajtest/helpers.py @@ -14,8 +14,10 @@ from doajtest.fixtures import ArticleFixtureFactory, ApplicationFixtureFactory from portality import core, dao, models from portality.core import app +from portality.dao import any_pending_tasks, query_data_tasks from portality.lib import paths, dates from portality.lib.dates import FMT_DATE_STD +from portality.lib.thread_utils import wait_until from portality.tasks.redis_huey import main_queue, long_running from portality.util import url_for @@ -83,6 +85,10 @@ def warmArticle(self): CREATED_INDICES = [] +def initialise_index(): + core.initialise_index(app, core.es_connection) + + def create_index(index_type): if index_type in CREATED_INDICES: return @@ -132,6 +138,7 @@ def create_app_patch(cls): "ES_RETRY_HARD_LIMIT": 0, "ES_BLOCK_WAIT_OVERRIDE": 0.5, "ES_READ_TIMEOUT": '5m', + 'ES_SOCKET_TIMEOUT': 5 * 60, "ELASTIC_SEARCH_DB": app.config.get('ELASTIC_SEARCH_TEST_DB'), 'ELASTIC_SEARCH_DB_PREFIX': create_es_db_prefix(cls), "FEATURES": app.config['VALID_FEATURES'], @@ -411,3 +418,20 @@ def login(app_client, username, password, follow_redirects=True): def logout(app_client, follow_redirects=True): return app_client.get(url_for('account.logout'), follow_redirects=follow_redirects) + + +def wait_until_no_es_incomplete_tasks(): + """ + + wait until no ES pending tasks and no data tasks is running + + created for make sure model.save() or model.delete() is completed + + if your data still can not be query, try Model.refresh() + + """ + + def _cond_fn(): + return not any_pending_tasks() and len(query_data_tasks(timeout='3m')) == 0 + + return wait_until(_cond_fn, 10, 0.2) diff --git a/doajtest/integration/__init__.py b/doajtest/integration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doajtest/integration/autocheck_resources/__init__.py b/doajtest/integration/autocheck_resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doajtest/integration/autocheck_resources/test_issn_org.py b/doajtest/integration/autocheck_resources/test_issn_org.py new file mode 100644 index 0000000000..9ddf23f452 --- /dev/null +++ b/doajtest/integration/autocheck_resources/test_issn_org.py @@ -0,0 +1,53 @@ +from doajtest.helpers import DoajTestCase + +from portality.autocheck.resource_bundle import Resource, ResourceBundle +from portality.autocheck.resources.issn_org import ISSNOrg + +######################################### +# We're expecting the following ISSNs to resolve to pages which embed schema.org data +# which have the following properties: +# +# * provisional - the ISSN is provisional, so the property mainEntityOfPage.version is not "Register" +# * archived - the ISSN is archived, so the property subjectOf contains at least one ArchiveComponent +# * registered - the ISSN is registered, so the property mainEntityOfPage.version is "Register" + +TESTS = { + "2786-5800": ["provisional"], + "2673-8198": ["archived"], + "1234-1231": ["registered"] +} + + +class TestISSNOrg(DoajTestCase): + def setUp(self): + super(TestISSNOrg, self).setUp() + + def tearDown(self): + super(TestISSNOrg, self).tearDown() + + def test_01_integrations(self): + resources = ResourceBundle() + issn_org = ISSNOrg(resources) + + for issn, expected in TESTS.items(): + data = issn_org.fetch(issn) + + if "provisional" in expected: + assert data is not None + assert data.is_registered() is False + + if "archived" in expected: + assert data is not None + assert len(data.archive_components) > 0 + + for ac in data.archive_components: + id = ac.get("holdingArchive", {}).get("@id") + tc = ac.get("temporalCoverage") + assert id is not None + assert tc is not None + + if "registered" in expected: + assert data is not None + assert data.is_registered() is True + + diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv index 1945cc9ad9..9965c80495 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv @@ -1,6 +1,6 @@ -test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order -1,none,ArgumentException,0,0,0,0,0,,,,, -2,no_role,,0,0,0,0,0,,,,, -3,admin,,1,1,1,1,1,1,2,3,4,5 -4,editor,,0,0,0,0,0,,,,, -5,assed,,0,0,0,0,0,,,,, +test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order +1,none,ArgumentException,0,0,0,0,0,0,,,,,, +2,no_role,,0,0,0,0,0,0,,,,,, +3,admin,,1,1,1,1,1,1,1,2,3,4,5,6 +4,editor,,0,0,0,0,0,0,,,,,, +5,assed,,0,0,0,0,0,0,,,,,, diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv index 419f22f89d..a8148032f9 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv @@ -1,32 +1,36 @@ -field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order -type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional -default,,,,,,,,,,,,, -,,,,,,,,,,,,, -values,,none,ArgumentException,,,,,,,,,, -values,,no_role,,,,,,,,,,, -values,,admin,,,,,,,,,,, -values,,editor,,,,,,,,,,, -values,,assed,,,,,,,,,,, -,,,,,,,,,,,,, -conditional raises,,none,ArgumentException,,,,,,,,,, -,,,,,,,,,,,,, -conditional todo_maned_stalled,,admin,,1,,,,,,,,, -conditional todo_maned_stalled,,!admin,,0,,,,,,,,, -,,,,,,,,,,,,, -conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,, -conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,, -,,,,,,,,,,,,, -conditional todo_maned_ready,,admin,,,,1,,,,,,, -conditional todo_maned_ready,,!admin,,,,0,,,,,,, -,,,,,,,,,,,,, -conditional todo_maned_completed,,admin,,,,,1,,,,,, -conditional todo_maned_completed,,!admin,,,,,0,,,,,, -,,,,,,,,,,,,, -conditional todo_maned_assign_pending,,admin,,,,,,1,,,,, -conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,, -,,,,,,,,,,,,, -conditional todo_maned_ready_order,,admin,,,,,,,1,,,, -conditional todo_maned_follow_up_old_order,,admin,,,,,,,,2,,, -conditional todo_maned_stalled_order,,admin,,,,,,,,,3,, -conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,4, -conditional todo_maned_completed_order,,admin,,,,,,,,,,,5 \ No newline at end of file +field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order +type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional +default,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,, +values,,none,ArgumentException,,,,,,,,,,,, +values,,no_role,,,,,,,,,,,,, +values,,admin,,,,,,,,,,,,, +values,,editor,,,,,,,,,,,,, +values,,assed,,,,,,,,,,,,, +,,,,,,,,,,,,,,, +conditional raises,,none,ArgumentException,,,,,,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_stalled,,admin,,1,,,,,,,,,,, +conditional todo_maned_stalled,,!admin,,0,,,,,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,,,, +conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_ready,,admin,,,,1,,,,,,,,, +conditional todo_maned_ready,,!admin,,,,0,,,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_completed,,admin,,,,,1,,,,,,,, +conditional todo_maned_completed,,!admin,,,,,0,,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_assign_pending,,admin,,,,,,1,,,,,,, +conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_new_update_request,,admin,,,,,,,1,,,,,, +conditional todo_maned_new_update_request,,!admin,,,,,,,0,,,,,, +,,,,,,,,,,,,,,, +conditional todo_maned_new_update_request_order,,admin,,,,,,,,1,,,,, +conditional todo_maned_ready_order,,admin,,,,,,,,,2,,,, +conditional todo_maned_follow_up_old_order,,admin,,,,,,,,,,3,,, +conditional todo_maned_stalled_order,,admin,,,,,,,,,,,4,, +conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,,,5, +conditional todo_maned_completed_order,,admin,,,,,,,,,,,,,6 \ No newline at end of file diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json index 28e9c66dc5..6625f298f2 100644 --- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json +++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json @@ -179,7 +179,7 @@ } }, { - "name": "todo_maned_ready_order", + "name": "todo_maned_new_update_request", "type": "conditional", "default": "", "values": { @@ -193,11 +193,40 @@ } } ] + }, + "0": { + "conditions": [ + { + "account": { + "nor": [ + "admin" + ] + } + } + ] } } }, { - "name": "todo_maned_follow_up_old_order", + "name": "todo_maned_new_update_request_order", + "type": "conditional", + "default": "", + "values": { + "1": { + "conditions": [ + { + "account": { + "or": [ + "admin" + ] + } + } + ] + } + } + }, + { + "name": "todo_maned_ready_order", "type": "conditional", "default": "", "values": { @@ -215,7 +244,7 @@ } }, { - "name": "todo_maned_stalled_order", + "name": "todo_maned_follow_up_old_order", "type": "conditional", "default": "", "values": { @@ -233,7 +262,7 @@ } }, { - "name": "todo_maned_assign_pending_order", + "name": "todo_maned_stalled_order", "type": "conditional", "default": "", "values": { @@ -251,7 +280,7 @@ } }, { - "name": "todo_maned_completed_order", + "name": "todo_maned_assign_pending_order", "type": "conditional", "default": "", "values": { @@ -267,6 +296,24 @@ ] } } + }, + { + "name": "todo_maned_completed_order", + "type": "conditional", + "default": "", + "values": { + "6": { + "conditions": [ + { + "account": { + "or": [ + "admin" + ] + } + } + ] + } + } } ] } \ No newline at end of file diff --git a/doajtest/mocks/autocheck_checkers.py b/doajtest/mocks/autocheck_checkers.py new file mode 100644 index 0000000000..fa7ada847d --- /dev/null +++ b/doajtest/mocks/autocheck_checkers.py @@ -0,0 +1,22 @@ +from portality.autocheck.checker import Checker + + +class AutocheckMockFactory(object): + @classmethod + def mock_autochecker(cls): + return MockChecker + + +class MockChecker(Checker): + __identity__ = "mock_annotator" + + def check(self, form, + jla, + autochecks, + resources, + logger): + autochecks.add_check(field="pissn", + original_value="1234-5678", + suggested_value="9876-5432", + advice="Change the issn", + reference_url="http://example.com/9876-5432") diff --git a/doajtest/mocks/autocheck_resource_bundle_Resource.py b/doajtest/mocks/autocheck_resource_bundle_Resource.py new file mode 100644 index 0000000000..1931433f19 --- /dev/null +++ b/doajtest/mocks/autocheck_resource_bundle_Resource.py @@ -0,0 +1,34 @@ +from doajtest.fixtures.resources import ResourcesFixtureFactory + +from portality.autocheck.resource_bundle import ResourceUnavailable + + +class ResourceBundleResourceMockFactory(object): + @classmethod + def no_contact_resource_fetch(cls, version=None, archive_components=None): + def mock(self, *args, **kwargs): + if self.__identity__ == "issn_org": + issn = None + if "issn" in kwargs: + issn = kwargs["issn"] + if len(args) > 0: + issn = args[0] + return ResourcesFixtureFactory.issn_org(issn=issn, version=version, archive_components=archive_components) + + return None + + return mock + + @classmethod + def fail_fetch(cls): + def mock(self, *args, **kwargs): + raise ResourceUnavailable() + + return mock + + @classmethod + def not_found_fetch(cls): + def mock(self, *args, **kwargs): + return None + + return mock \ No newline at end of file diff --git a/doajtest/selenium_helpers.py b/doajtest/selenium_helpers.py index b3f5e5ea21..80d9e2e42e 100644 --- a/doajtest/selenium_helpers.py +++ b/doajtest/selenium_helpers.py @@ -1,14 +1,13 @@ import datetime import logging -import time import multiprocessing +import time from multiprocessing import Process, freeze_support from typing import TYPE_CHECKING import selenium from selenium import webdriver -from selenium.common import StaleElementReferenceException, ElementClickInterceptedException -from selenium.webdriver import DesiredCapabilities +from selenium.common.exceptions import StaleElementReferenceException, ElementClickInterceptedException from selenium.webdriver.common.by import By from doajtest.fixtures.url_path import URL_LOGOUT diff --git a/doajtest/testbook/admin_article_metadata_form/admin_article_metadata_form.yml b/doajtest/testbook/admin_article_metadata_form/admin_article_metadata_form.yml index ca85eded33..d3f7df9563 100644 --- a/doajtest/testbook/admin_article_metadata_form/admin_article_metadata_form.yml +++ b/doajtest/testbook/admin_article_metadata_form/admin_article_metadata_form.yml @@ -2,118 +2,138 @@ suite: Admin Article Metadata Form testset: Admin Article Metadata Form tests: -- title: Preparation - context: - role: Admin - setup: - - Ensure you own a journal with ISSNs 1234-5678 and 9876-5432 and no articles - attached - - Ensure you are an owner of a journal with ISSN 0000-0000 that is not in - DOAJ - steps: - - step: Upload set of test articles - resource: /xml_upload_test_package/admin_metadata_form_test_pack.xml - results: - - 3 Articles are uploaded and attached to the journal + - title: Preparation + context: + role: Admin + setup: + - Ensure you own a journal with ISSNs 1234-5678 and 9876-5432 and no articles + attached + - Ensure you are an owner of a journal with ISSN 0000-0000 that is not in + DOAJ + steps: + - step: Upload set of test articles + resource: /xml_upload_test_package/admin_metadata_form_test_pack.xml + results: + - 3 Articles are uploaded and attached to the journal -- title: Updating data other then DOI and Fulltext URL - context: - role: Admin - depends: - - suite: Admin Article Metadata Form - testset: Admin Article Metadata Form - test: Preparation - steps: - - step: Go to admin journal and articles search, choose Articles from the "Journals - vs Articles" facets at the left - results: - - Each article in the search results has "Delete this article | Edit this article" - option in the lower right corner - - step: Find "Success 300" article - - step: Click "Edit this article" - results: - - The Admin Metadata Form for "Success 300" article is opened in the new window - - step: Check if PISSN and EISSN dropdowns have 1234-5678 and 9876-5432 options - - step: Check that PISSN and EISSN dropdowns have not 0000-0000 option - - step: Check the Authors section - results: - - There is only one author and no "Remove Author" button - - step: Click "Add Author" - results: - - New empty Author Subform appears, each has "Remove Author" option - - step: Click "Remove Author" under one of the authors - results: - - The author is removed from the list - - step: Change the title to "Success 400" - - step: Add an author filling in any string as a name, an affiliation and an orcid - id in the correct form, for example https://orcid.org/0001-1234-1234-444X - - step: Change all the other fields except doi and fulltext URL to something easily - recognisable - - step: Click "Update Article" button - results: - - Green banner saying "Article created/updated" is shown at the top of the page. - - step: Search for "Success 400" article again - - step: Close the page with the Article Metadata Form - - step: Click "Edit the article" - results: - - Confirm all the metadata is updated + - title: Updating data other then DOI and Fulltext URL + context: + role: Admin + depends: + - suite: Admin Article Metadata Form + testset: Admin Article Metadata Form + test: Preparation + steps: + - step: Go to admin journal and articles search, choose Articles from the "Journals + vs Articles" facets at the left + results: + - Each article in the search results has "Delete this article | Edit this article" + option in the lower right corner + - step: Find "Success 300" article + - step: Click "Edit this article" + results: + - The Admin Metadata Form for "Success 300" article is opened in the new window + - step: Check if PISSN and EISSN dropdowns have 1234-5678 and 9876-5432 options + - step: Check that PISSN and EISSN dropdowns have not 0000-0000 option + - step: Check the Authors section + results: + - There is only one author and no "Remove Author" button + - step: Click "Add Author" + results: + - New empty Author Subform appears, each has "Remove Author" option + - step: Click "Remove Author" under one of the authors + results: + - The author is removed from the list + - step: Change the title to "Success 400" + - step: Add an author filling in any string as a name, an affiliation and an orcid + id in the correct form, for example https://orcid.org/0001-1234-1234-444X + - step: Change all the other fields except doi and fulltext URL to something easily + recognisable + - step: Click "Update Article" button + results: + - Green banner saying "Article created/updated" is shown at the top of the page. + - step: Search for "Success 400" article again + - step: Close the page with the Article Metadata Form + - step: Click "Edit the article" + results: + - Confirm all the metadata is updated -- title: Checking data validation - context: - role: Admin - depends: - - suite: Admin Article Metadata Form - testset: Admin Article Metadata Form - test: Updating data other then DOI and Fulltext URL - steps: - - step: Open "Success 400" article metadata form - - step: 'Change DOI to any string in invalid format, eg: 0000' - - step: 'Change author''s orcid_id to the wrong format (eg: "0000")' - - step: Click "Update Article" button - results: - - Error "Invalid DOI. A DOI can optionally start with a prefix (such as "doi:"), - followed by "10." and the remainder of the identifier" under DOI field and "Invalid - ORCID iD. Please enter your ORCID iD as a full URL of the form https://orcid.org/0000-0000-0000-0000" - is shown under orcid_id field - - step: Close the form - results: - - Confirm the metadata of "Success 400" article has not change + - title: Checking data validation + context: + role: Admin + depends: + - suite: Admin Article Metadata Form + testset: Admin Article Metadata Form + test: Updating data other then DOI and Fulltext URL + steps: + - step: Open "Success 400" article metadata form + - step: 'Change DOI to any string in invalid format, eg: 0000' + - step: 'Change author''s orcid_id to the wrong format (eg: "0000")' + - step: Click "Update Article" button + results: + - Error "Invalid DOI. A DOI can optionally start with a prefix (such as "doi:"), + followed by "10." and the remainder of the identifier" under DOI field and "Invalid + ORCID iD. Please enter your ORCID iD as a full URL of the form https://orcid.org/0000-0000-0000-0000" + is shown under orcid_id field + - step: Close the form + results: + - Confirm the metadata of "Success 400" article has not change -- title: Change DOI and Fulltext URL - context: - role: Admin - depends: - - suite: Admin Article Metadata Form - testset: Admin Article Metadata Form - test: Updating data other then DOI and Fulltext URL - steps: - - step: Open a "Success 400" article metadata form - - step: Change DOI to 10.1234/200 and Fulltext URL to http://doaj.org/testing/500.pdf - - step: Click "Update article" button - results: - - The red banner "Article could not be updated, as it matches another existing - article. Please check your metadata, and contact us if you cannot resolve the - issue yourself." - - step: Change DOI to 10.1234/500 and Fulltext URL to http://doaj.org/testing/200.pdf - - step: Click "Update article" button - results: - - The red banner "Article could not be updated, as it matches another existing - article. Please check your metadata, and contact us if you cannot resolve the - issue yourself." - - step: Change DOI back to 10.1234/400 and Fulltext URL to http://doaj.org/testing/500.pdf - - step: Click "Update article" button - results: - - Green banner "Article created/updated" is shown at the top page - - step: Open the "Success 400" landing page - results: - - Confirm that Fulltext URL is changed to http://doaj.org/testing/500.pdf - - step: Go back to the article metadata form - - step: Change DOI to 10.1234/500 - - step: Click "Update article" button - results: - - Green banner "Article created/updated" is shown at the top page - - step: Open the "Success 400" landing page - results: - - Confirm that Fulltext URL is changed to http://doaj.org/testing/500.pdf - - Confirm only one article with DOI 10.1234/500 and fulltext URL http://doaj.org/testing/500.pdf - exists + - title: Change DOI and Fulltext URL + context: + role: Admin + depends: + - suite: Admin Article Metadata Form + testset: Admin Article Metadata Form + test: Updating data other then DOI and Fulltext URL + steps: + - step: Open a "Success 400" article metadata form + - step: Change DOI to 10.1234/200 and Fulltext URL to http://doaj.org/testing/500.pdf + - step: Click "Update article" button + results: + - The red banner "Article could not be updated, as it matches another existing + article. Please check your metadata, and contact us if you cannot resolve the + issue yourself." + - step: Change DOI to 10.1234/500 and Fulltext URL to http://doaj.org/testing/200.pdf + - step: Click "Update article" button + results: + - The red banner "Article could not be updated, as it matches another existing + article. Please check your metadata, and contact us if you cannot resolve the + issue yourself." + - step: Change DOI back to 10.1234/400 and Fulltext URL to http://doaj.org/testing/500.pdf + - step: Click "Update article" button + results: + - Green banner "Article created/updated" is shown at the top page + - step: Open the "Success 400" landing page + results: + - Confirm that Fulltext URL is changed to http://doaj.org/testing/500.pdf + - step: Go back to the article metadata form + - step: Change DOI to 10.1234/500 + - step: Click "Update article" button + results: + - Green banner "Article created/updated" is shown at the top page + - step: Open the "Success 400" landing page + results: + - Confirm that Fulltext URL is changed to http://doaj.org/testing/500.pdf + - Confirm only one article with DOI 10.1234/500 and fulltext URL http://doaj.org/testing/500.pdf + exists + + - title: Check button linking to the article's page + context: + role: Admin + steps: + - step: Click "See this article in doaj" button under the article's title on the left + results: + - The article's public page is opened in a new tab + - On the article's public page at the top the button "Edit this article" is shown + - step: Click "Edit this article" button + results: + - The artcile's admin metadata form is opened in a new tab with correct data + - step: Confirm above buttons are shown only for admin user. Log out from admin account and log in as publisher + - step: Navigate to the article's public page as a publisher + results: + - The button "Edit this article" is not displayed + - step: Log out + - step: Navigate to the article's public page as an anonymous user + results: + - The button "Edit this article" is not displayed diff --git a/doajtest/testbook/article_metadata_upload_form/article_metadata_upload_form.yml b/doajtest/testbook/article_metadata_upload_form/article_metadata_upload_form.yml index 2e6d391ca1..2808e1065d 100644 --- a/doajtest/testbook/article_metadata_upload_form/article_metadata_upload_form.yml +++ b/doajtest/testbook/article_metadata_upload_form/article_metadata_upload_form.yml @@ -10,7 +10,7 @@ tests: - step: Do not fill in any data yet. Click "Add Article" button at the bottom of the page results: - - Black popup "Please fill that field in" below the Title field is shown + - You are scrolled back up to the Title field - Title field is focussed (it is possible to type straight into the field without the need to click in it) - step: "In \"Article Title\" textbox enter valid article title\n (any string of\ @@ -21,10 +21,10 @@ tests: - 'Red error: ''You must provide the Full-Text URL or the DOI'' appears under the URL field and the DOI field' - Red error "Please provide at least one author" under Authors fields appears - - Red error "Either this field or Journal ISSN (online version) is required" under - "Journal ISSN (print version)" appears - - Red error "Either this field or Journal ISSN (print version) is required" under - "Journal ISSN (online version)" appears + - Red error "Either this field or Online ISSN is required" under + "ISSN, Print" appears + - Red error "Either this field or Print ISSN is required" under + "ISSN, Online" appears - step: In Authors section click on orange "Add more authors" button results: - One row for author is added @@ -55,11 +55,10 @@ tests: - step: Select 2 identical ISSNs - step: Click "Add Article" button results: - - 'Errors: "This field must contain a different value to the field ''eissn''" - and "This field must contain a different value to the field ''pissn''" are shown - below ISSNs inputs' + - 'The error: "The Print and Online ISSNs supplied are identical. If you supply 2 ISSNs they must be different." + is shown below each ISSN field.' - step: Choose different Online ISSN - step: Click "Add Article" button results: - - 'At the top of the page you see the confirmation message: Article created/updated' + - 'At the top of the page you see the confirmation message: Article created/updated (Dismiss)' - https://docs.google.com/spreadsheets/d/1KGv7DEwocDvdbT8giN9Aw2EWtZGzKympouEw_uR2q18/edit#gid=782367369 diff --git a/doajtest/testbook/article_xml_upload/article_doaj_xml_upload.yml b/doajtest/testbook/article_xml_upload/article_doaj_xml_upload.yml index 6a896f9daf..37852ae306 100644 --- a/doajtest/testbook/article_xml_upload/article_doaj_xml_upload.yml +++ b/doajtest/testbook/article_xml_upload/article_doaj_xml_upload.yml @@ -21,8 +21,8 @@ tests: based (e.g. an image file or a PDF) - step: Click "Upload" results: - - A flash message with appropriate text appears at the top of the screen indicating an error has occurred - - Your file is shown in the "History of uploads" with status "processing failed" + - A flash message with appropriate text appears at the top of the screen, indicating an error has occurred + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes". Check that the explanation link goes to a suitable reason and resolution for the problem. @@ -38,8 +38,8 @@ tests: - step: Choose "DOAJ Native XML" from "Format of the file" dropdown - step: Click "Upload" results: - - A flash message with appropriate text appears at the top of the screen indicating an error has occurred - - Your file is shown in the "History of uploads" with status "processing failed" + - A flash message with appropriate text appears at the top of the screen, indicating an error has occurred + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes" (you may need to reload the page). Check that the explanation link goes to a suitable reason and resolution for the problem. @@ -55,8 +55,8 @@ tests: - step: Choose "DOAJ Native XML" from "Format of the file" dropdown - step: Click "Upload" results: - - A flash message with appropriate text appears at the top of the screen indicating an error has occurred - - Your file is shown in the "History of uploads" with status "processing failed" + - A flash message with appropriate text appears at the top of the screen, indicating an error has occurred + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes". Check that the explanation link goes to a suitable reason and resolution for the problem. @@ -75,12 +75,11 @@ tests: results: - 'A flash message appears at the top of the screen: File uploaded and waiting to be processed. Check back here for updates.' - - Your file is shown in the "History of uploads" with status "pending" - - step: wait a short amount of time for the job to process, then reload the page - (do not re-submit the form data). If the job remains in "pending", reload the + - Click away to another tab and then back - your file is shown in the "History of uploads" with the status "pending" + - step: wait a short amount of time for the job to process, then click away and then back again. If the job remains in "pending", reload the page until the status changes. results: - - Your file is shown in the "History of uploads" with status "processing failed" + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes". Check that the explanation link goes to a suitable reason and resolution for the problem. @@ -100,10 +99,9 @@ tests: - step: Click "Upload" results: - 'A flash message appears at the top of the screen indicating a successful upload: - File uploaded and waiting to be processed. Check back here for updates.(Dismiss)' - - Your file is shown in the "History of uploads" with status "pending" - - step: wait a short amount of time for the job to process, then reload the page - (do not re-submit the form data). If the job remains in "pending", reload the + File uploaded and waiting to be processed. Check back here for updates. (Dismiss)' + - Click away to another tab and then back - your file is shown in the "History of uploads" with status "pending" + - step: wait a short amount of time for the job to process, then click away and then back. If the job remains in "pending", reload the page until the status changes. results: - Your file is shown in the "History of uploads" with status "processing failed" @@ -111,14 +109,14 @@ tests: a suitable reason and resolution for the problem. - step: click on "(show error details)" for the record in the "History of uploads" results: - - Additional error details are shown, indicating that the publisher does not own + - Clicking 'show error details' shows information indicating that the publisher does not own ISSNs "0000-0000" and "0000-000X" - title: Upload a file containing ISSN that has been withdrawn context: role: publisher steps: - - step: Ensure that the publisher owns a journal with ISSN 0000-0000 that is not + - step: Ensure that the publisher owns a journal with ISSN 0000-1111 that is not in DOAJ (ie. has been withdrawn) - step: Go to the "Upload Article XML" tab in the "Publisher Area" - step: Choose "DOAJ Native XML" from "Format of the file" dropdown @@ -128,19 +126,17 @@ tests: - step: Click "Upload" results: - 'A flash message appears at the top of the screen indicating a successful upload: - File uploaded and waiting to be processed. Check back here for updates.(Dismiss)' - - Your file is shown in the "History of uploads" with status "pending" - - step: wait a short amount of time for the job to process, then reload the page - (do not re-submit the form data). If the job remains in "pending", reload the + File uploaded and waiting to be processed. Check back here for updates. (Dismiss)' + - Click away to another tab and then back - your file is shown in the "History of uploads" with the status "pending" + - step: wait a short amount of time for the job to process, then click away and then back. If the job remains in "pending", reload the page until the status changes. results: - - Your file is shown in the "History of uploads" with status "processing failed" + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes". Check that the explanation link goes to a suitable reason and resolution for the problem. - step: click on "(show error details)" for the record in the "History of uploads" results: - - Additional error details are shown, indicating that the journal has been withdrawn - from DOAJ and therefore the article cannot be accepted + - extra information indicates that ISSN is not in DOAJ - title: Upload a file containing ISSNs not previously seen in DOAJ context: @@ -156,13 +152,12 @@ tests: - step: Click "Upload" results: - 'A flash message appears at the top of the screen indicating a successful upload: - File uploaded and waiting to be processed. Check back here for updates.(Dismiss)' - - Your file is shown in the "History of uploads" with status "pending" - - step: wait a short amount of time for the job to process, then reload the page - (do not re-submit the form data). If the job remains in "pending", reload the + File uploaded and waiting to be processed. Check back here for updates. (Dismiss)' + - Click away to another tab and then back - your file is shown in the "History of uploads" with the status "pending" + - step: wait a short amount of time for the job to process, then click away and then back. If the job remains in "pending", reload the page until the status changes. results: - - Your file is shown in the "History of uploads" with status "processing failed" + - Your file is shown in the "History of uploads" with the status "processing failed" and a suitable entry in the "Notes". Check that the explanation link goes to a suitable reason and resolution for the problem. - step: click on "(show error details)" for the record in the "History of uploads" @@ -214,9 +209,9 @@ tests: resource: /xml_upload_test_package/DOAJ/successful.xml - step: Click "Upload" results: - - A flash message appears at the top of the screen indicating a successful upload + - A flash message appears at the top of the screen, indicating a successful upload File uploaded and waiting to be processed. Check back here for updates.(Dismiss) - - Your file is shown in the "History of uploads" with status "pending" + - Your file is shown in the "History of uploads" with the status "pending" - step: wait a short amount of time for the job to process, then reload the page (do not re-submit the form data). If the job remains in "pending", reload the page until the status changes. @@ -239,9 +234,9 @@ tests: resource: /xml_upload_test_package/DOAJ/update.xml - step: Click "Upload" results: - - A flash message appears at the top of the screen indicating a successful upload + - A flash message appears at the top of the screen, indicating a successful upload File uploaded and waiting to be processed. Check back here for updates.(Dismiss) - - Your file is shown in the "History of uploads" with status "pending" + - Your file is shown in the "History of uploads" with the status "pending" - step: wait a short amount of time for the job to process, then reload the page (do not re-submit the form data). If the job remains in "pending", reload the page until the status changes. @@ -258,16 +253,16 @@ tests: role: publisher steps: - step: Run through test 7 and 8 to upload and update a new article - - step: 'Make sure there are no other articles in the databse with the DOI: 10.1234/100. + - step: 'Make sure there are no other articles in the database with the DOI: 10.1234/100. Delete those first.' - step: Go to the "Upload Article XML" tab in the "Publisher Area" - step: In the box "Provide a URL where we can download the XML", enter the URL below resource: /xml_upload_test_package/DOAJ/successful.xml - step: Click "Upload" results: - - A flash message appears at the top of the screen indicating that the file reference + - A flash message appears at the top of the screen, indicating that the file reference was successfully received - - Your file is shown in the "History of uploads" with status "pending" + - Your file is shown in the "History of uploads" with the status "pending" - step: wait a short amount of time for the job to process, then reload the page (do not re-submit the form data). If the job remains in "pending", reload the page until the status changes. @@ -361,4 +356,4 @@ tests: results: - Related background job is found - status is "complete" - - Outcome Status is "success" \ No newline at end of file + - Outcome Status is "success" diff --git a/doajtest/testbook/autocheck/application_submission_workflow.yml b/doajtest/testbook/autocheck/application_submission_workflow.yml new file mode 100644 index 0000000000..00742c53d9 --- /dev/null +++ b/doajtest/testbook/autocheck/application_submission_workflow.yml @@ -0,0 +1,40 @@ +suite: Autocheck +testset: Application Submission Workflow +tests: + +- title: Submit application for autochecking + steps: + - step: To create the publisher and admin accounts required for this test, and to prepare the environment, + use the testdrive setup linked below + path: /testdrive/autocheck_application_submission + results: + - you receive a page which contains at least a set of admin account credentials and publisher account credentials + - step: Log in with a publisher account + path: /account/login + - step: Go to the application form + path: /apply/ + - step: | + Complete the application form as you like, but ensure to set the following values: + * ISSN (print): 1848-3380 + * ISSN (online): 0005-1144 + * Archiving policy: CLOCKSS, LOCKSS + - step: Submit the application + results: + - The application is successfully submitted + - step: Log out as a publisher and log in as an administrator + - step: Go to the background jobs view + results: + - A background job is present for the application autochecks for the application you just submitted (it is probably + the top job, but if not limit by action "application_autochecks" to find it) + - The status of the job is either "queued" or "completed". If it is "queued" wait for it to finish (you will need to refresh + the page to see the status change) + - It should move to "completed" fairly quickly, if it does not the test may have failed + - step: Go to the application search and find the application you just submitted (e.g. sort by date applied, descending, and it + should be the first result) + - step: Open the application for reviewing and scroll through it + results: + - There is text at the top which tells you when the Autochecks were made, dated to today, and an option to Hide All Autochecks + - The ISSN (print) field has an autocheck annotation + - The ISSN (online) field has an autocheck annotation + - The Archiving Policy question has autocheck annotations for all selected services + - step: Delete the accounts you created for this test by clicking the "Teardown" link in the testdrive page you used to create them \ No newline at end of file diff --git a/doajtest/testbook/autocheck/autocheck.yml b/doajtest/testbook/autocheck/autocheck.yml new file mode 100644 index 0000000000..e13965b5ac --- /dev/null +++ b/doajtest/testbook/autocheck/autocheck.yml @@ -0,0 +1,122 @@ +suite: Autocheck +testset: Autocheck +tests: + +- title: Preparation + context: + role: Any Authenticated User + steps: + - step: Activate the testdrive setup for this test suite, by using the following url + path: /testdrive/autocheck + results: + - You receive the testdrive setup details, including an admin account for you to use, + and an application and journal which have been autochecked for testing + - step: Log into DOAJ using the admin credentials supplied by the testdrive. You may want to do + this in a private browsing window, so you can remain logged in to your own account in your + main window. + path: /account/login + results: + - You are logged in as an admin user + - step: You may now proceed to the tests below + +- title: Application autochecks + context: + role: Admin + depends: + - suite: Autocheck + testset: Autocheck + test: Preparation + steps: + - step: Go to the Application admin_url supplied by the testdrive setup + results: + - You are looking at a test Application created for this test + - step: Scroll down to find the ISSN (print) and ISSN (online) fields + results: + - The Print ISSN field is annotated with a green tick, and text which says that ISSN + is fully registered at ISSN.org + - The Electronic ISSN field is annotated with an orange cross, which says the ISSN is not + registered at ISSN.org + - step: Click on one of the "see record" links in the annotation + results: + - The link is opened in a new window/tab + - You are taken to the ISSN.org record. Note that for the purposes of this test, this + is a random record on ISSN.org, unrelated to the actual record in DOAJ. + - step: Close the ISSN.org window/tab and return to the application form + - step: Scroll to the "Best Practice" section of the application form, and look at the "Long-term preservation services" question + results: + - "4 checks are visible attached to this question, for: CLOCKSS, LOCKSS, PMC and PKP PN" + - CLOCKSS is annotated with a green tick, saying it is archived + - LOCKSS is annotated with a red cross, saying it is not current + - PMC is annotated with a grey info symbol, saying it is not currently recorded by Keepers + - PKP PN is annotated with a red cross, saying it is not archived + - step: Click on one of the "see record" links in the annotation + results: + - The link is opened in a new window/tab + - You are taken to the ISSN.org record. Note that for the purposes of this test, this + is a random record on ISSN.org, unrelated to the actual record in DOAJ. + - step: Close the ISSN.org window/tab and return to the application form + - step: Scroll to the top of the application form + results: + - There is text which tells you when the Autochecks were made, and an option to Hide All Autochecks + - step: Click "Hide All Autochecks", then scroll through the form + results: + - The autochecks are all hidden + - step: Return to the top of the application form and click "Show All Autochecks", then scroll through the form + results: + - The autochecks are all visible again + +- title: Journal autochecks + context: + role: Admin + depends: + - suite: Autocheck + testset: Autocheck + test: Preparation + steps: + - step: Go to the Journal admin_url supplied by the testdrive setup + results: + - You are looking at a test Journal created for this test + - step: Scroll down to find the ISSN (print) and ISSN (online) fields + results: + - The Print ISSN field is annotated with a green tick, and text which says that ISSN + is fully registered at ISSN.org + - The Electronic ISSN field is annotated with an red cross, which says the ISSN is not + found at ISSN.org + - step: Click on one of the "see record" links in the annotation + results: + - The link is opened in a new window/tab + - You are taken to the ISSN.org record. Note that for the purposes of this test, this + is a random record on ISSN.org, unrelated to the actual record in DOAJ. + - step: Close the ISSN.org window/tab and return to the journal form + - step: Scroll to the "Best Practice" section of the application form, and look at the "Long-term preservation services" question + results: + - "4 checks are visible attached to this question, for: CLOCKSS, LOCKSS, PMC and PKP PN" + - CLOCKSS is annotated with a green tick, saying it is archived + - LOCKSS is annotated with a red cross, saying it is not current + - PMC is annotated with a grey info symbol, saying it is not currently recorded by Keepers + - PKP PN is annotated with a red cross, saying it is not archived + - step: Click on one of the "see record" links in the annotation + results: + - The link is opened in a new window/tab + - You are taken to the ISSN.org record. Note that for the purposes of this test, this + is a random record on ISSN.org, unrelated to the actual record in DOAJ. + - step: Close the ISSN.org window/tab and return to the journal form + - step: Scroll to the top of the journal form + results: + - There is text which tells you when the Autochecks were made, and an option to Hide All Autochecks + - step: Click "Hide All Autochecks", then scroll through the form + results: + - The autochecks are all hidden + - step: Return to the top of the journal form and click "Show All Autochecks", then scroll through the form + results: + - The autochecks are all visible again + +- title: Teardown + context: + role: Any Authenticated User + steps: + - step: Close your private browsing window you used for the test + - step: Click the "Teardown" url supplied by the testdrive setup + results: + - You receive a success notification in your browser window + - step: You can now close the testdrive browser tab/window \ No newline at end of file diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml index 8bc3564a6d..96c7b171c4 100644 --- a/doajtest/testbook/dashboards/maned_todo.yml +++ b/doajtest/testbook/dashboards/maned_todo.yml @@ -21,41 +21,75 @@ tests: in the ready state, an application in your maned group which is in the completed state, an application in your maned group which has not had an associate editor assigned, an application created over 10 weeks ago in your maned group, an application in your maned group which has not been updated for 8 weeks." + - "The user account should be assigned to one update request which was created over 4 weeks ago" steps: - step: log in as a managing editor - step: Go to the maned dashboard page path: /dashboard results: - - You can see 13 applications in your priority list + - You can see 16 applications in your priority list - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), editor items (actions related to teams you are the editor for) and associate items (actions related to applications which are assigned specifically to you for review). + - Your highest priority item (first in the list) is for an update request which was submitted this month - At least one of your priority items is for an application which is older than 10 weeks (it should indicate that it is for your maned group) - At least one of your priority items is for an application which has been inactive (stalled) for more than 8 weeks (it should indicate that it is for your maned group) - At least one of your priority items is for an application in the state ready (it should indicate that it is for your maned group) - At least one of your priority items is for an application in the completed state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) - At least one of your priority items is for an application in the pending state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group) + - Your lowest priority item (last in the list) is for an update request which was submitted this month - step: click on the managing editor's ready application - step: Change the application status to "Accepted" and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 12 applications in your priority list + - You can see 15 applications in your priority list - The application you have just edited has disappeared from your priority list - step: click on the [in progress] stalled managing editor's application - step: make any minor adjustment to the metadata and save - step: close the tab, return to the dashboard and reload the page results: - - You can see 11 applications in your priority list + - You can see 14 applications in your priority list - The application you just edited has disappeared from your priority list - step: click on the "completed" maned application - step: Change the application to "ready" status - step: close the tab, return to the dashboard and reload the page results: - - You can still see 11 applications in your priority list + - You can still see 14 applications in your priority list - The completed application you just moved to ready is now in your priority list as a ready application - step: click on the pending managing editor's application - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select) - step: close the tab, return to the dashboard and reload the page results: - - You have 10 applications left in your todo list - - The pending application you just edited is no longer visible \ No newline at end of file + - You have 13 applications left in your todo list + - The pending application you just edited is no longer visible + + - title: Filtering the todo list + context: + role: admin + testdrive: todo_maned_editor_associate + setup: + - Use the todo_maned_editor_associate testdrive to setup for this test, OR follow the setup from the previous test + steps: + - step: log in as a managing editor + - step: Go to the maned dashboard page + path: /dashboard + results: + - You can see 16 applications in your priority list + - Your highest priority item (first in the list) is for an update request which was submitted last month + - Your lowest priority item (last in the list) is for an update request which was submitted this month + - On the top right of the todo list are a set of filter buttons "Show all", "New Applications" and "Update Requests" + - The "Show all" button is highlighted + - step: click on the "New Applications" filter button + results: + - You can see 14 applications in your priority list + - The update requests which were on the previous screen are no longer visible + - The "New Applications" filter button is now highlighted + - step: click on the "Update Request" filter button + results: + - You can see 12application in your priority list + - Your highest priority item (first in the list) is for an update request which was submitted last month + - Your lowest priority item (last in the list) is for an update request which was submitted this month + - The "Update Request" filter button is now highlighted + - step: click the "Show all" filter button + results: + - You are back to the original display, containing both applications and update requests \ No newline at end of file diff --git a/doajtest/testbook/journal_form/maned_form.yml b/doajtest/testbook/journal_form/maned_form.yml index 935fe4504f..7206d57653 100644 --- a/doajtest/testbook/journal_form/maned_form.yml +++ b/doajtest/testbook/journal_form/maned_form.yml @@ -126,3 +126,22 @@ tests: - step: Attempt to paste the value (use separate editor) results: - Correct value is pasted +- title: Check button linking to the article's page + context: + role: Admin + steps: + - step: Click "See this journal in doaj" button under the journal's title on the left + results: + - The journal's public page is opened in a new tab + - On the journal's public page at the top the button "Edit this journal" is shown + - step: Click "Edit this journal" button + results: + - The journal's admin metadata form is opened in a new tab with correct data + - step: Confirm above buttons are shown only for admin user. Log out from admin account and log in as publisher + - step: Navigate to the article's public page as a publisher + results: + - The button "Edit this journal" is not displayed + - step: Log out + - step: Navigate to the article's public page as an anonymous user + results: + - The button "Edit this journal" is not displayed diff --git a/doajtest/testbook/new_application_form/maned_form.yml b/doajtest/testbook/new_application_form/maned_form.yml index 10c524df2a..82c7fdf388 100644 --- a/doajtest/testbook/new_application_form/maned_form.yml +++ b/doajtest/testbook/new_application_form/maned_form.yml @@ -48,7 +48,10 @@ tests: results: - A message at the top of the form tells you that you can only choose one or two subject classifications - - step: Remove one subject classification and 'Save' + - step: Remove one subject classification + - step: Click "Unlock & Close" + results: Confirmation dialog is displayed warning you about unsaved changes + - step: Click "Stay on Page" and then 'Save' results: - The form saves - The changes you applied, both to the form, and in the functionality box, have diff --git a/doajtest/testdrive/autocheck.py b/doajtest/testdrive/autocheck.py new file mode 100644 index 0000000000..7e9bf4fbbe --- /dev/null +++ b/doajtest/testdrive/autocheck.py @@ -0,0 +1,171 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from doajtest.fixtures.v2.applications import ApplicationFixtureFactory +from doajtest.fixtures.v2.journals import JournalFixtureFactory +from portality import models +from datetime import datetime +from portality.autocheck.checkers.issn_active import ISSNActive, ISSNChecker +from portality.autocheck.resources.issn_org import ISSNOrgData +from portality.autocheck.checkers.keepers_registry import KeepersRegistry +from portality.bll import DOAJ +from flask import url_for +from portality.core import app + +class Autocheck(TestDrive): + + def setup(self) -> dict: + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "Admin " + un, [constants.ROLE_ADMIN]) + acc.set_password(pw) + acc.save() + + ################################################## + ## Setup and Application with the following features + ## + ## - Print ISSN registered at ISSN.org + ## - Electronic ISSN not registered at ISSN.org + ## - 3 preservation services: + ## - CLOCKSS - currently archived + ## - LOCKSS - not currently archived + ## - PMC - not registered + source = ApplicationFixtureFactory.make_application_source() + ap = models.Application(**source) + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION + ap.remove_current_journal() + ap.remove_related_journal() + apbj = ap.bibjson() + apbj.set_preservation(["CLOCKSS", "LOCKSS", "PMC", "PKP PN"], "http://policy.example.com") + ap.set_id(ap.makeid()) + ap.save() + + bj = ap.bibjson() + pissn = bj.get_one_identifier(bj.P_ISSN) + eissn = bj.get_one_identifier(bj.E_ISSN) + + thisyear = datetime.utcnow().year + + pissn_data = ISSNOrgData({ + "mainEntityOfPage": { + "version": "Register" # this means the ISSN is registered at ISSN.org + }, + "subjectOf": [ + { + "@type": "ArchiveComponent", + "holdingArchive": { + "@id": "http://issn.org/organization/keepers#clockss" + }, + "temporalCoverage": "2022/" + str(thisyear) + }, + { + "@type": "ArchiveComponent", + "holdingArchive": { + "@id": "http://issn.org/organization/keepers#lockss" + }, + "temporalCoverage": "2019/2020" + } + ] + }) + + eissn_data = ISSNOrgData({ + "mainEntityOfPage": { + "version": "Pending" # this means the ISSN is not registered at ISSN.org + }, + "subjectOf": [ + { + "@type": "ArchiveComponent", + "holdingArchive": { + "@id": "http://issn.org/organization/keepers#clockss" + }, + "temporalCoverage": "2022/" + str(thisyear) + }, + { + "@type": "ArchiveComponent", + "holdingArchive": { + "@id": "http://issn.org/organization/keepers#lockss" + }, + "temporalCoverage": "2019/2020" + } + ] + }) + + old_retrieve_from_source = ISSNChecker.retrieve_from_source + ISSNChecker.retrieve_from_source = lambda *args, **kwargs: ( + eissn, + "https://portal.issn.org/resource/ISSN/2682-4396", + eissn_data, + False, + pissn, + "https://portal.issn.org/resource/ISSN/2682-4396", + pissn_data, + False) + + acSvc = DOAJ.autochecksService( + autocheck_plugins=[ + # (journal, application, plugin) + (True, True, ISSNActive), + (True, True, KeepersRegistry) + ] + ) + ac1 = acSvc.autocheck_application(ap) + + ################################################## + ## Setup a Journal with the following features + ## + ## - Print ISSN registered at ISSN.org + ## - Electronic ISSN not found + ## - 3 preservation services: + ## - CLOCKSS - currently archived + ## - LOCKSS - not currently archived + ## - PMC - not registered + + source = JournalFixtureFactory.make_journal_source() + j = models.Journal(**source) + j.remove_current_application() + j.set_id(ap.makeid()) + j.save() + + bj = j.bibjson() + pissn = bj.get_one_identifier(bj.P_ISSN) + eissn = bj.get_one_identifier(bj.E_ISSN) + + ISSNChecker.retrieve_from_source = lambda *args, **kwargs: ( + eissn, + "https://portal.issn.org/resource/ISSN/9999-000X", + None, # Don't pass in any data, so we get the Not Found response + False, + pissn, + "https://portal.issn.org/resource/ISSN/2682-4396", + pissn_data, + False) + + ac2 = acSvc.autocheck_journal(j) + + ISSNChecker.retrieve_from_source = old_retrieve_from_source + + return { + "account": { + "username": acc.id, + "password": pw + }, + "application": { + "id": ap.id, + "admin_url": app.config.get("BASE_URL") + url_for("admin.application", application_id=ap.id) + }, + "journal": { + "id": j.id, + "admin_url": app.config.get("BASE_URL") + url_for("admin.journal_page", journal_id=j.id) + }, + "autocheck": { + "application": ac1.id, + "journal": ac2.id + } + } + + def teardown(self, params): + models.Account.remove_by_id(params["account"]["username"]) + models.Application.remove_by_id(params["application"]["id"]) + models.Journal.remove_by_id(params["journal"]["id"]) + models.Autocheck.remove_by_id(params["autocheck"]["application"]) + models.Autocheck.remove_by_id(params["autocheck"]["journal"]) + return {"status": "success"} diff --git a/doajtest/testdrive/autocheck_application_submission.py b/doajtest/testdrive/autocheck_application_submission.py new file mode 100644 index 0000000000..fd42eff85d --- /dev/null +++ b/doajtest/testdrive/autocheck_application_submission.py @@ -0,0 +1,50 @@ +from portality import constants +from doajtest.testdrive.factory import TestDrive +from portality import models + +class AutocheckApplicationSubmission(TestDrive): + """ + setup for the testbook test autocheck/application_submission_workflow + """ + + def setup(self) -> dict: + # admin + un = self.create_random_str() + pw1 = self.create_random_str() + admin = models.Account.make_account(un + "@example.com", un, "Admin " + un, [constants.ROLE_ADMIN]) + admin.set_password(pw1) + admin.save() + + # publisher + un = self.create_random_str() + pw2 = self.create_random_str() + pub = models.Account.make_account(un + "@example.com", un, "Publisher " + un, [constants.ROLE_PUBLISHER]) + pub.set_password(pw2) + pub.save() + + # ensure that the issns that we are going to use in the test are removed from the database + js = models.Journal.find_by_issn("1848-3380") + if js is not None: + for j in js: + j.delete() + + js = models.Journal.find_by_issn("0005-1144") + if js is not None: + for j in js: + j.delete() + + return { + "admin": { + "username": admin.id, + "password": pw1 + }, + "publisher": { + "username": pub.id, + "password": pw2 + } + } + + def teardown(self, params): + models.Account.remove_by_id(params["admin"]["username"]) + models.Account.remove_by_id(params["publisher"]["username"]) + return {"status": "success"} diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py index a6efd6f747..1fa8ff936e 100644 --- a/doajtest/testdrive/todo_maned_editor_associate.py +++ b/doajtest/testdrive/todo_maned_editor_associate.py @@ -152,18 +152,42 @@ def build_maned_applications(un, eg, owner, eponymous_group): "title": un + " Maned Low Priority Pending Application" }] + lmur = build_application(un + " Last Month Maned Update Request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, + editor_group=eponymous_group.name, owner=owner, update_request=True) + lmur.save() + + tmur = build_application(un + " This Month Maned Update Request", 0, 0, constants.APPLICATION_STATUS_UPDATE_REQUEST, + editor_group=eponymous_group.name, owner=owner, update_request=True) + tmur.save() + + apps["update_request"] = [ + { + "id": lmur.id, + "title": un + " Last Month Maned Update Request" + }, + { + "id": tmur.id, + "title": un + " This Month Maned Update Request" + } + ] + return apps -def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None, owner=None): +def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None, owner=None, update_request=False): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.bibjson().title = title ap.set_id(ap.makeid()) - ap.remove_current_journal() - ap.remove_related_journal() del ap.bibjson().discontinued_date - ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION + + if update_request: + ap.application_type = constants.APPLICATION_TYPE_UPDATE_REQUEST + else: + ap.remove_current_journal() + ap.remove_related_journal() + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION + ap.set_last_manual_update(dates.before(datetime.utcnow(), lmu_diff)) ap.set_created(dates.before(datetime.utcnow(), cd_diff)) ap.set_date_applied(dates.before(datetime.utcnow(), cd_diff)) diff --git a/doajtest/unit/api_tests/test_api_account.py b/doajtest/unit/api_tests/test_api_account.py index 5cc1b1b8ea..0ea640259f 100644 --- a/doajtest/unit/api_tests/test_api_account.py +++ b/doajtest/unit/api_tests/test_api_account.py @@ -1,5 +1,6 @@ from flask import Response +from doajtest import helpers from doajtest.helpers import DoajTestCase, with_es from portality import models from portality.core import load_account_for_login_manager @@ -10,6 +11,7 @@ class TestAPIClient(DoajTestCase): @classmethod def setUpClass(cls): super(TestAPIClient, cls).setUpClass() + helpers.initialise_index() # Turn off debug so we're allowed to add these routes after the app has been used in other tests cls.app_test.debug = False diff --git a/doajtest/unit/autocheck_resources/__init__.py b/doajtest/unit/autocheck_resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doajtest/unit/autocheck_resources/test_issn_org.py b/doajtest/unit/autocheck_resources/test_issn_org.py new file mode 100644 index 0000000000..180945325c --- /dev/null +++ b/doajtest/unit/autocheck_resources/test_issn_org.py @@ -0,0 +1,35 @@ +from doajtest.helpers import DoajTestCase +from doajtest.fixtures import ApplicationFixtureFactory, IssnOrgFixtureFactory +from doajtest.mocks.autocheck_resource_bundle_Resource import ResourceBundleResourceMockFactory + +from portality.autocheck.checkers.issn_active import ISSNActive +from portality import models +from portality.autocheck.resource_bundle import Resource, ResourceBundle +from portality.autocheck.resources.issn_org import ISSNOrg + +import responses # mocks for the requests library + + +class TestISSNOrg(DoajTestCase): + def setUp(self): + super(TestISSNOrg, self).setUp() + + def tearDown(self): + super(TestISSNOrg, self).tearDown() + + @responses.activate + def test_01_issn_fetch(self): + resources = ResourceBundle() + issn_org = ISSNOrg(resources) + + rsp1 = responses.Response( + method="GET", + url=issn_org.reference_url("1234-5678"), + body=IssnOrgFixtureFactory.web_page_body() + ) + responses.add(rsp1) + + data = issn_org.fetch("1234-5678") + + assert data is not None + assert data.is_registered() diff --git a/doajtest/unit/autocheck_resources/test_resource.py b/doajtest/unit/autocheck_resources/test_resource.py new file mode 100644 index 0000000000..98ebf55174 --- /dev/null +++ b/doajtest/unit/autocheck_resources/test_resource.py @@ -0,0 +1,35 @@ +from doajtest.helpers import DoajTestCase +from doajtest.fixtures import ApplicationFixtureFactory, IssnOrgFixtureFactory +from doajtest.mocks.autocheck_resource_bundle_Resource import ResourceBundleResourceMockFactory + +from portality.autocheck.checkers.issn_active import ISSNActive +from portality import models +from portality.autocheck.resource_bundle import Resource, ResourceBundle, ResourceUnavailable +from portality.autocheck.resources.issn_org import ISSNOrg + +import responses # mocks for the requests library + + +class TestResource(DoajTestCase): + def setUp(self): + super(TestResource, self).setUp() + + def tearDown(self): + super(TestResource, self).tearDown() + + def test_01_resource_bundle(self): + resources = ResourceBundle() + + resinst = Resource(resources) + resinst.fetch_fresh = lambda *args, **kwargs: {"test": "data"} + data = resinst.fetch("arbitrary_resource") + + assert data == {"test": "data"} + assert resources.get(resinst.make_resource_id()) == {"test": "data"} + + def test_02_resource_unavailable(self): + resources = ResourceBundle() + + resinst = Resource(resources) + with self.assertRaises(ResourceUnavailable): + data = resinst.fetch("arbitrary_resource") diff --git a/doajtest/unit/autocheckers/__init__.py b/doajtest/unit/autocheckers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doajtest/unit/autocheckers/test_issn_active.py b/doajtest/unit/autocheckers/test_issn_active.py new file mode 100644 index 0000000000..b83eb8f316 --- /dev/null +++ b/doajtest/unit/autocheckers/test_issn_active.py @@ -0,0 +1,105 @@ +from doajtest.helpers import DoajTestCase +from doajtest.fixtures import ApplicationFixtureFactory +from doajtest.mocks.autocheck_resource_bundle_Resource import ResourceBundleResourceMockFactory + +from portality.autocheck.checkers.issn_active import ISSNActive +from portality import models +from portality.autocheck.resource_bundle import Resource, ResourceBundle + + +class TestISSNActive(DoajTestCase): + def setUp(self): + self.old_fetch = Resource.fetch + super(TestISSNActive, self).setUp() + + def tearDown(self): + Resource.fetch = self.old_fetch + super(TestISSNActive, self).tearDown() + + def test_01_issn_fetch_fail(self): + Resource.fetch = ResourceBundleResourceMockFactory.fail_fetch() + + issn_active = ISSNActive() + + form = { + "pissn": "1234-5678", + "eissn": "9876-5432" + } + + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + + autochecks = models.Autocheck() + resources = ResourceBundle() + + issn_active.check(form, app, autochecks, resources, logger=lambda x: x) + + assert len(autochecks.checks) == 2 + for anno in autochecks.checks: + assert anno.get("advice") == issn_active.UNABLE_TO_ACCESS + + def test_02_not_found(self): + Resource.fetch = ResourceBundleResourceMockFactory.not_found_fetch() + + issn_active = ISSNActive() + + form = { + "pissn": "1234-5678", + "eissn": "9876-5432" + } + + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + + autochecks = models.Autocheck() + resources = ResourceBundle() + + issn_active.check(form, app, autochecks, resources, logger=lambda x: x) + + assert len(autochecks.checks) == 2 + for anno in autochecks.checks: + assert anno.get("advice") == issn_active.NOT_FOUND + + def test_03_fully_validated(self): + Resource.fetch = ResourceBundleResourceMockFactory.no_contact_resource_fetch(version="Register") + + issn_active = ISSNActive() + + form = { + "pissn": "1234-5678", + "eissn": "9876-5432" + } + + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + + autochecks = models.Autocheck() + resources = ResourceBundle() + + issn_active.check(form, app, autochecks, resources, logger=lambda x: x) + + assert len(autochecks.checks) == 2 + for anno in autochecks.checks: + assert anno.get("advice") == issn_active.FULLY_VALIDATED + + def test_04_not_validated(self): + Resource.fetch = ResourceBundleResourceMockFactory.no_contact_resource_fetch(version="Pending") + + issn_active = ISSNActive() + + form = { + "pissn": "1234-5678", + "eissn": "9876-5432" + } + + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + + autochecks = models.Autocheck() + resources = ResourceBundle() + + issn_active.check(form, app, autochecks, resources, logger=lambda x: x) + + assert len(autochecks.checks) == 2 + for anno in autochecks.checks: + assert anno.get("advice") == issn_active.NOT_VALIDATED \ No newline at end of file diff --git a/doajtest/unit/autocheckers/test_keepers_registry.py b/doajtest/unit/autocheckers/test_keepers_registry.py new file mode 100644 index 0000000000..7e49f4612f --- /dev/null +++ b/doajtest/unit/autocheckers/test_keepers_registry.py @@ -0,0 +1,61 @@ +from doajtest.helpers import DoajTestCase +from doajtest.fixtures import ApplicationFixtureFactory +from doajtest.mocks.autocheck_resource_bundle_Resource import ResourceBundleResourceMockFactory + +from portality.autocheck.checkers.keepers_registry import KeepersRegistry +from portality import models +from portality.autocheck.resource_bundle import Resource, ResourceBundle + + +class TestKeepersRegistry(DoajTestCase): + def setUp(self): + self.old_fetch = Resource.fetch + super(TestKeepersRegistry, self).setUp() + + def tearDown(self): + Resource.fetch = self.old_fetch + super(TestKeepersRegistry, self).tearDown() + + # Note that we do not test the failure to fetch cases as they are already covered + # by the ISSNActive tests + + def test_01_registered(self): + Resource.fetch = ResourceBundleResourceMockFactory.no_contact_resource_fetch(archive_components={ + "CLOCKSS": True, + "LOCKSS": True, + "Internet Archive": False + }) + + kr = KeepersRegistry() + + form = { + "pissn": "1234-5678", + "eissn": "9876-5432", + "preservation_service": ["LOCKSS", "Internet Archive", "PKP PN"] + } + + source = ApplicationFixtureFactory.make_application_source() + app = models.Application(**source) + + autochecks = models.Autocheck() + resources = ResourceBundle() + + kr.check(form, app, autochecks, resources, logger=lambda x: x) + + assert len(autochecks.checks) == 3 + + checks = [False, False, False] + for check in autochecks.checks: + if check["context"]["service"] == "LOCKSS": + assert check["advice"] == kr.PRESENT + checks[0] = True + + if check["context"]["service"] == "Internet Archive": + assert check["advice"] == kr.OUTDATED + checks[1] = True + + if check["context"]["service"] == "PKP PN": + assert check["advice"] == kr.MISSING + checks[2] = True + + assert all(checks) \ No newline at end of file diff --git a/doajtest/unit/resources/annotation_examples.csv b/doajtest/unit/resources/annotation_examples.csv new file mode 100644 index 0000000000..c6766743fe --- /dev/null +++ b/doajtest/unit/resources/annotation_examples.csv @@ -0,0 +1,80 @@ +2587-2559 +1300-3747 +2544-963X +2579-4965 +1848-9184 +2217-5563 +1816-6326 +2079-1771 +2519-2752 +2560-5011 +1680-2004 +2073-753X +2288-2294 +2093-0569 +2645-6109 +2617-9555 +2313-3783 +2663-5828 +1791-4884 +2652-3647 +2528-9462 +2223-2125 +2384-8901 +2447-018X +2354-7006 +2352-9148 +0807-7096 +2175-5787 +1699-7778 +2344-5416 +1300-3747 +2352-9148 +1816-6326 +2110-5820 +2579-4965 +2073-753X +2288-2294 +2544-963X +2110-5820 +2383-4420 +2287-9129 +2587-2559 +2544-963X +2579-4965 +1647-0818 +1549-4497 +1647-0818 +2367-5144 +2078-5127 +1542-6300 +1022-9825 +1548-7733 +2316-9125 +2767-1402 +1678-4766 +2391-6702 +1848-9184 +1504-3029 +2519-2752 +1848-3380 +1816-6326 +2217-5563 +2384-8901 +2447-018X +2386-4303 +1647-0818 +1769-7298 +2663-2365 +1022-9825 +2652-3647 +2354-7006 +1816-6326 +2663-5828 +1406-0132 +2078-5127 +1848-9184 +2078-5127 +2413-9432 +2079-1771 +3434-3434 diff --git a/doajtest/unit/resources/issn_org_web_page.html b/doajtest/unit/resources/issn_org_web_page.html new file mode 100644 index 0000000000..4a10e7bfb7 --- /dev/null +++ b/doajtest/unit/resources/issn_org_web_page.html @@ -0,0 +1,695 @@ + + + + + + + + + + + + + ISSN 2682-4396 (Print) | Egyptian Journal of Medical Research | The ISSN Portal + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + + +
+
+
+ + +
+
+
+
+ + +
+
+ +
+
+ +
+
+
+

+

+
+ + + +
+
+ +
+
+
HomeRecord
+
+ +
+
+ +
+
+ Advanced search + ROAD search The Keepers search
+
+ + +
+ +
+
+
+ + +
+
+
+
+ + +
+ + + + +
+ + +
+
+ +
+ +
+

Identifiers

+
+
+
ISSN :

2682-4396

Linking ISSN (ISSN-L): 2682-4396

+
+ +
+
+ + +
+

Links

+
+
+

Wikidata: www.wikidata.or ... http://www.wikidata.org/entity/Q96697037

+

FATCAT: fatcat.wiki/con ... https://fatcat.wiki/container/ma2yplmihvaihj76dwbnrno7m4 +

+

Google: www.google.com/ ... https://www.google.com/search?q=ISSN+%222682-4396%22

+

Bing: www.bing.com/se ... https://www.bing.com/search?q=ISSN+%222682-4396%22

+

Yahoo: search.yahoo.co ... https://search.yahoo.com/search?p=ISSN%20%222682-4396%22

+

CROSSREF: search.crossref ... https://search.crossref.org/?q=+2682-4396

+ +
+ +
+
+
+ + +
+
+ + +
+

+
+ + + + +
+ + + +
+ +
+
+ +
+
Key-title  

Egyptian Journal of Medical Research (Print)

+
+
+ +
+
Identifiers
+
+
+
ISSN :

2682-4396

Linking ISSN (ISSN-L): 2682-4396

+
+
+
+ +
+
+ + +
Resource information
+
+
+
+ +
+ + + + +
+
+
+ + +
+

Title proper: Egyptian Journal of Medical Research

Country: Egypt

Medium: Print

+ +
+ +
+ + +
+
+
+
+
+
+
+ + + + +
+
Record information
+
+
+ +

Last modification date: 22/11/2022

Type of record: Confirmed

ISSN Center responsible of the record: ISSN EGYPT

+
+
+
+ +
+
Links
+
+
+

Wikidata: www.wikidata.or ... http://www.wikidata.org/entity/Q96697037

+

FATCAT: fatcat.wiki/con ... https://fatcat.wiki/container/ma2yplmihvaihj76dwbnrno7m4 +

+

Wikidata: www.wikidata.or ... http://www.wikidata.org/entity/Q96697037

+

FATCAT: fatcat.wiki/con ... https://fatcat.wiki/container/ma2yplmihvaihj76dwbnrno7m4 +

+

Google: www.google.com/ ... https://www.google.com/search?q=ISSN+%222682-4396%22

+

Bing: www.bing.com/se ... https://www.bing.com/search?q=ISSN+%222682-4396%22

+

Yahoo: search.yahoo.co ... https://search.yahoo.com/search?p=ISSN%20%222682-4396%22

+

CROSSREF: search.crossref ... https://search.crossref.org/?q=+2682-4396

+ +
+
+
+ + + + +
+ + +
+
+ +
+ + + + +
.........
+
+
+

My Tools

+
+
+ Share +
+ + + Print + + + + Display linked data + + + + + +
+
+

Enjoy Premium features

+ Unlock functions +
+ +
+ +
+
+ +
+ + + + + +
+
+
+
+ + + + + + + + + + + + +
+
+
+
+
+ + + +
+ + + diff --git a/doajtest/unit/test_bll_autochecks.py b/doajtest/unit/test_bll_autochecks.py new file mode 100644 index 0000000000..2c0a669f1f --- /dev/null +++ b/doajtest/unit/test_bll_autochecks.py @@ -0,0 +1,84 @@ +import time + +from doajtest.helpers import DoajTestCase +from portality import models +from portality.bll import DOAJ + +from doajtest.fixtures import ApplicationFixtureFactory, JournalFixtureFactory +from doajtest.mocks.autocheck_resource_bundle_Resource import ResourceBundleResourceMockFactory +from doajtest.mocks.autocheck_checkers import AutocheckMockFactory + +from portality.autocheck.resource_bundle import Resource + + +class TestBLLAutochecks(DoajTestCase): + + def setUp(self): + mock_fetch = ResourceBundleResourceMockFactory.no_contact_resource_fetch() + self.old_fetch = Resource.fetch + Resource.fetch = mock_fetch + + super(TestBLLAutochecks, self).setUp() + + def tearDown(self): + Resource.fetch = self.old_fetch + super(TestBLLAutochecks, self).tearDown() + + def test_01_annotate_application(self): + source = ApplicationFixtureFactory.make_application_source() + application = models.Application(**source) + application.save(blocking=True) + + ma = AutocheckMockFactory.mock_autochecker() + anno_svc = DOAJ.autochecksService([(True, True, ma)]) + anno_svc.autocheck_application(application) + + time.sleep(2) + + application = models.Application.pull(application.id) + autocheck = models.Autocheck.for_application(application.id) + + # assert application.application_status == constants.APPLICATION_STATUS_PENDING + assert autocheck is not None + assert autocheck.application == application.id + assert len(autocheck.checks) == 1 + + def test_02_annotate_journal(self): + source = JournalFixtureFactory.make_journal_source() + journal = models.Journal(**source) + journal.save(blocking=True) + + ma = AutocheckMockFactory.mock_autochecker() + anno_svc = DOAJ.autochecksService([(True, True, ma)]) + anno_svc.autocheck_journal(journal) + + time.sleep(2) + + journal = models.Journal.pull(journal.id) + autocheck = models.Autocheck.for_journal(journal.id) + + # assert application.application_status == constants.APPLICATION_STATUS_PENDING + assert autocheck is not None + assert autocheck.journal == journal.id + assert len(autocheck.checks) == 1 + + def test_03_dismiss_undismiss(self): + autocheck = models.Autocheck() + autocheck.application = "test_application" + check = autocheck.add_check("field", "original", "suggested", "advice", "reference", {"context": "here"}, "test") + autocheck.save(blocking=True) + + anno_svc = DOAJ.autochecksService() + anno_svc.dismiss(autocheck.id, check.get("id")) + + time.sleep(2) + + ac2 = models.Autocheck.for_application("test_application") + assert ac2.checks[0].get("dismissed") is True + + anno_svc.undismiss(autocheck.id, check.get("id")) + + time.sleep(2) + + ac3 = models.Autocheck.for_application("test_application") + assert ac3.checks[0].get("dismissed", False) is False \ No newline at end of file diff --git a/doajtest/unit/test_bll_reject_application.py b/doajtest/unit/test_bll_reject_application.py index cf208dbb21..486b6a1793 100644 --- a/doajtest/unit/test_bll_reject_application.py +++ b/doajtest/unit/test_bll_reject_application.py @@ -2,7 +2,8 @@ from parameterized import parameterized -from portality import constants +from doajtest import helpers +from portality import constants, dao from doajtest.fixtures import ApplicationFixtureFactory, AccountFixtureFactory, JournalFixtureFactory from doajtest.helpers import DoajTestCase, load_from_matrix from portality.bll import DOAJ @@ -84,7 +85,8 @@ def test_01_reject_application(self, name, application, application_status, acco svc.reject_application(ap, acc, provenance, note=thenote) else: svc.reject_application(ap, acc, provenance, note=thenote) - time.sleep(2) + helpers.wait_until_no_es_incomplete_tasks() + dao.refresh() ####################################### ## Check @@ -96,8 +98,8 @@ def test_01_reject_application(self, name, application, application_status, acco # check the updated and manually updated date are essentially the same (they can theoretically differ # by a small amount just based on when they are set) - assert wait_until( - lambda: (ap2.last_updated_timestamp - ap2.last_manual_update_timestamp).total_seconds() <= 1.0,) + updated_spread = abs((ap2.last_updated_timestamp - ap2.last_manual_update_timestamp).total_seconds()) + assert updated_spread <= 1.0 if current_journal == "yes" and journal is not None: j2 = Journal.pull(journal.id) diff --git a/doajtest/unit/test_bll_todo_top_todo_assed.py b/doajtest/unit/test_bll_todo_top_todo_assed.py index 59448569cc..83260249e4 100644 --- a/doajtest/unit/test_bll_todo_top_todo_assed.py +++ b/doajtest/unit/test_bll_todo_top_todo_assed.py @@ -1,16 +1,13 @@ -from time import sleep - -from parameterized import parameterized from combinatrix.testintegration import load_parameter_sets - from doajtest.fixtures import ApplicationFixtureFactory, AccountFixtureFactory, EditorGroupFixtureFactory -from doajtest.helpers import DoajTestCase +from doajtest.helpers import DoajTestCase, wait_until_no_es_incomplete_tasks +from parameterized import parameterized from portality import constants from portality import models from portality.bll import DOAJ from portality.bll import exceptions -from portality.lib.paths import rel2abs from portality.lib import dates +from portality.lib.paths import rel2abs def load_cases(): @@ -90,7 +87,8 @@ def test_top_todo(self, name, kwargs): # an application that is otherwise normal self.build_application("assed_all_applications", 2 * w, 2 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) - sleep(2) + wait_until_no_es_incomplete_tasks() + models.Application.refresh() # size = int(size_arg) size=25 diff --git a/doajtest/unit/test_bll_todo_top_todo_editor.py b/doajtest/unit/test_bll_todo_top_todo_editor.py index 8f799b1d9c..f5f6d4a2e7 100644 --- a/doajtest/unit/test_bll_todo_top_todo_editor.py +++ b/doajtest/unit/test_bll_todo_top_todo_editor.py @@ -1,16 +1,13 @@ -from time import sleep - -from parameterized import parameterized from combinatrix.testintegration import load_parameter_sets - from doajtest.fixtures import ApplicationFixtureFactory, AccountFixtureFactory, EditorGroupFixtureFactory -from doajtest.helpers import DoajTestCase +from doajtest.helpers import DoajTestCase, wait_until_no_es_incomplete_tasks +from parameterized import parameterized from portality import constants from portality import models from portality.bll import DOAJ from portality.bll import exceptions -from portality.lib.paths import rel2abs from portality.lib import dates +from portality.lib.paths import rel2abs def load_cases(): @@ -93,7 +90,8 @@ def assign_pending(ap): self.build_application("editor_assign_pending", 2 * w, 2 * w, constants.APPLICATION_STATUS_PENDING, apps, additional_fn=assign_pending) - sleep(2) + wait_until_no_es_incomplete_tasks() + models.Application.refresh() # size = int(size_arg) size=25 diff --git a/doajtest/unit/test_bll_todo_top_todo_maned.py b/doajtest/unit/test_bll_todo_top_todo_maned.py index 2e68313fe3..5322a8c9ae 100644 --- a/doajtest/unit/test_bll_todo_top_todo_maned.py +++ b/doajtest/unit/test_bll_todo_top_todo_maned.py @@ -1,16 +1,13 @@ -from time import sleep - -from parameterized import parameterized from combinatrix.testintegration import load_parameter_sets - from doajtest.fixtures import ApplicationFixtureFactory, AccountFixtureFactory, EditorGroupFixtureFactory -from doajtest.helpers import DoajTestCase +from doajtest.helpers import DoajTestCase, wait_until_no_es_incomplete_tasks +from parameterized import parameterized from portality import constants from portality import models from portality.bll import DOAJ from portality.bll import exceptions -from portality.lib.paths import rel2abs from portality.lib import dates +from portality.lib.paths import rel2abs def load_cases(): @@ -43,7 +40,8 @@ def test_top_todo(self, name, kwargs): "todo_maned_follow_up_old", "todo_maned_ready", "todo_maned_completed", - "todo_maned_assign_pending" + "todo_maned_assign_pending", + "todo_maned_new_update_request" ] category_args = { @@ -98,6 +96,10 @@ def assign_pending(ap): self.build_application("maned_assign_pending", 4 * w, 4 * w, constants.APPLICATION_STATUS_PENDING, apps, assign_pending) + # an update request + self.build_application("maned_update_request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, apps, + update_request=True) + # Applications that should never be reported ############################################ @@ -158,7 +160,8 @@ def noeditorgroup(ap): # counter to maned_assign_pending self.build_application("no_assed", 3 * w, 3 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, assign_pending) - sleep(2) + wait_until_no_es_incomplete_tasks() + models.Application.refresh() # size = int(size_arg) size=25 @@ -194,7 +197,7 @@ def noeditorgroup(ap): else: # the todo item is not positioned at all assert len(positions.get(k, [])) == 0 - def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additional_fn=None): + def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additional_fn=None, update_request=False): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.set_id(id) @@ -202,6 +205,13 @@ def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additio ap.set_date_applied(dates.before_now(cd_diff)) ap.set_application_status(status) + if update_request: + ap.application_type = constants.APPLICATION_TYPE_UPDATE_REQUEST + else: + ap.remove_current_journal() + ap.remove_related_journal() + ap.application_type = constants.APPLICATION_TYPE_NEW_APPLICATION + if additional_fn is not None: additional_fn(ap) diff --git a/doajtest/unit/test_forms_application_processors_admin.py b/doajtest/unit/test_forms_application_processors_admin.py index 08dd74443e..032a4775e6 100644 --- a/doajtest/unit/test_forms_application_processors_admin.py +++ b/doajtest/unit/test_forms_application_processors_admin.py @@ -1,3 +1,4 @@ +from doajtest import helpers from doajtest.fixtures import JournalFixtureFactory from doajtest.helpers import DoajTestCase from portality import models, constants @@ -13,6 +14,11 @@ class TestPublicApplicationProcessorAdmin(DoajTestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + helpers.initialise_index() + def setUp(self): super(TestPublicApplicationProcessorAdmin, self).setUp() diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index 06175e6d76..bf626153d7 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1642,6 +1642,58 @@ def test_38_language_lax(self): a2 = models.Application(**asource) assert a2.bibjson().language.pop() == 'interpretive dance' + def test_39_autochecks(self): + a = models.Autocheck() + a.application = "1234" + a.journal = "9876" + a.add_check("field", "original", "suggested", "advice", "http://ref.com", {"context": "here"}, "checker") + + assert a.application == "1234" + assert a.journal == "9876" + assert len(a.checks) == 1 + assert len(a.checks_raw) == 1 + + check = a.checks[0] + assert check["field"] == "field" + assert check["original_value"] == "original" + assert check["suggested_value"] == ["suggested"] + assert check["advice"] == "advice" + assert check["reference_url"] == "http://ref.com" + assert check["context"] == {"context": "here"} + assert check["checked_by"] == "checker" + assert check["id"] is not None + assert "dismissed" not in check + + a.add_check("field", "original", "suggested", "advice", "http://ref.com", {"context": "here"}, "checker") + assert len(a.checks) == 1 + + a.add_check("field2", "original", "suggested", "advice", "http://ref.com", {"context": "here"}, "checker") + assert len(a.checks) == 2 + + a.dismiss(check["id"]) + check = a.checks[0] + assert check["dismissed"] is True + + a.undismiss(check["id"]) + check = a.checks[0] + assert "dismissed" not in check + + def test_40_autocheck_retrieves(self): + a = models.Autocheck() + a.application = "1234" + a.add_check("field", "original", "suggested", "advice", "http://ref.com", {"context": "here"}, "checker") + a.save(blocking=True) + + ap2 = models.Autocheck.for_application("1234") + assert ap2.application == "1234" + + a = models.Autocheck() + a.journal = "9876" + a.add_check("field", "original", "suggested", "advice", "http://ref.com", {"context": "here"}, "checker") + a.save(blocking=True) + + ap2 = models.Autocheck.for_journal("9876") + assert ap2.journal == "9876" class TestAccount(DoajTestCase): def test_get_name_safe(self): diff --git a/doajtest/unit/test_openurl.py b/doajtest/unit/test_openurl.py index 7ab023d962..08bed5dc61 100644 --- a/doajtest/unit/test_openurl.py +++ b/doajtest/unit/test_openurl.py @@ -116,7 +116,10 @@ def test_03_openurl_multiple_prarameters(self): volume="1")) assert resp.status_code == 302 - assert resp.location == url_for('doaj.toc', identifier=j_matching.id, volume="1") + + # FIXME: we have removed volumes from the ToC so we can't resolve the volume in the query + # assert resp.location == url_for('doaj.toc', identifier=j_matching.id, volume="1") + assert resp.location == url_for('doaj.toc', identifier=j_matching.id) # A query without genre to show it's the default resp = t_client.get(url_for('openurl.openurl', diff --git a/doajtest/unit/test_task_preservation.py b/doajtest/unit/test_task_preservation.py index d8d62d87e8..faf3e15a06 100644 --- a/doajtest/unit/test_task_preservation.py +++ b/doajtest/unit/test_task_preservation.py @@ -3,13 +3,15 @@ import requests from io import BytesIO from unittest.mock import patch -from doajtest.helpers import DoajTestCase +from doajtest.helpers import DoajTestCase, login from doajtest.mocks.preservation import PreservationMock from portality.tasks import preservation from portality.core import app from portality.lib import dates from werkzeug.datastructures import FileStorage from portality.models.article import Article +from portality.models import Account +from portality.ui.messages import Messages def mock_pull_by_key(key, value): @@ -232,3 +234,15 @@ def test_get_article_info(self): assert issn == "2051-5960" assert article_id == "00003741594643f4996e2555a01e03c7" assert metadata_json["bibjson"]["identifier"][0]["id"] == "10.1186/s40478-018-0619-9" + + def test_empty_file(self): + admin_account = Account.make_account(email="admin@test.com", username="admin", name="Admin", roles=["admin"]) + admin_account.set_password('password123') + admin_account.save() + + with self.app_test.test_client() as t_client: + login(t_client, "admin", "password123") + response = t_client.post('/publisher/preservation', data={}) + with t_client.session_transaction() as session: + flash_messages = session.get('_flashes') + assert any(msg[1] == Messages.PRESERVATION_NO_FILE for msg in flash_messages) diff --git a/doajtest/unit/test_tasks_anon_export.py b/doajtest/unit/test_tasks_anon_export.py index 6fbbaa16b2..c17e158139 100644 --- a/doajtest/unit/test_tasks_anon_export.py +++ b/doajtest/unit/test_tasks_anon_export.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from doajtest.helpers import DoajTestCase +from doajtest.helpers import DoajTestCase, StoreLocalPatcher from doajtest.unit_tester import bgtask_tester from portality.models import BackgroundJob, Account from portality.store import StoreLocal @@ -12,11 +12,24 @@ class TestAnonExport(DoajTestCase): + def setUp(self): + super().setUp() + self.store_local_patcher = StoreLocalPatcher() + self.store_local_patcher.setUp(self.app_test) + + def tearDown(self): + super().tearDown() + self.store_local_patcher.tearDown(self.app_test) + def test_execute(self): # prepare test data + BackgroundJob.destroy_index() + Account.destroy_index() BackgroundJob.save_all((BackgroundJob() for _ in range(3)), blocking=True) Account.save_all((Account() for _ in range(2)), blocking=True) + BackgroundJob.refresh() + Account.refresh() new_background_jobs = list(BackgroundJob.scroll()) new_accounts = list(Account.scroll()) @@ -46,7 +59,14 @@ def test_execute(self): rows = data_str.strip().split('\n') # Filter out the index: directives, leaving the actual record data - json_rows = list(filter(lambda j: len(json.loads(j).keys()) > 1, rows)) + json_rows = (json.loads(j) for j in rows) + json_rows = filter(lambda j: len(j.keys()) > 1, json_rows) + # drop additional background job record for AnonExportBackgroundTask execute + json_rows = (j for j in json_rows if ( + j.get('action') != 'anon_export' and + j.get('status') != 'processing' + )) + json_rows = list(json_rows) if target_name.startswith('background_job'): test_data_list = new_background_jobs @@ -58,7 +78,7 @@ def test_execute(self): print(f'number of rows have been saved to store: [{target_name}] {len(json_rows)}') self.assertEqual(len(json_rows), len(test_data_list)) - self.assertIn(test_data_list[0].id, [json.loads(j)['id'] for j in json_rows]) + self.assertIn(test_data_list[0].id, [j['id'] for j in json_rows]) else: print(f'empty archive {target_name}') diff --git a/doajtest/unit/test_tasks_application_autochecks.py b/doajtest/unit/test_tasks_application_autochecks.py new file mode 100644 index 0000000000..e04f0fc87c --- /dev/null +++ b/doajtest/unit/test_tasks_application_autochecks.py @@ -0,0 +1,63 @@ +import time + +from doajtest.helpers import DoajTestCase, StoreLocalPatcher +from doajtest.unit_tester import bgtask_tester +from portality.background import BackgroundApi +from portality.core import app +from portality.store import StoreFactory +from portality.tasks import application_autochecks +from doajtest.fixtures import ApplicationFixtureFactory +from portality import models, constants +from portality.bll.services import autochecks +from doajtest.mocks.autocheck_checkers import AutocheckMockFactory + + +class TestApplicationAutochecks(DoajTestCase): + + def setUp(self): + self.old_plugins = autochecks.AUTOCHECK_PLUGINS + autochecks.AUTOCHECK_PLUGINS = [ + (True, True, AutocheckMockFactory.mock_autochecker()) + ] + super(TestApplicationAutochecks, self).setUp() + + def tearDown(self): + autochecks.AUTOCHECK_PLUGINS = self.old_plugins + super(TestApplicationAutochecks, self).tearDown() + + def test_01_application_not_found(self): + user = app.config.get("SYSTEM_USERNAME") + job = application_autochecks.ApplicationAutochecks.prepare(user, application="123456") + task = application_autochecks.ApplicationAutochecks(job) + BackgroundApi.execute(task) + + assert not job.is_failed() + assert job.outcome_status == "success" + assert job.status == "complete" + + def test_02_application_annotated_and_status_set(self): + user = app.config.get("SYSTEM_USERNAME") + + source = ApplicationFixtureFactory.make_application_source() + application = models.Application(**source) + application.set_application_status(constants.APPLICATION_STATUS_POST_SUBMISSION_REVIEW) + application.save(blocking=True) + + job = application_autochecks.ApplicationAutochecks.prepare(user, application=application.id, status_on_complete=constants.APPLICATION_STATUS_PENDING) + task = application_autochecks.ApplicationAutochecks(job) + BackgroundApi.execute(task) + + assert not job.is_failed() + assert job.outcome_status == "success" + assert job.status == "complete" + + time.sleep(2) + application = models.Application.pull(application.id) + assert application.application_status == constants.APPLICATION_STATUS_PENDING + + # check that the autocheck annotation is present + ac = models.Autocheck.for_application(application.id) + assert ac is not None + assert len(ac.checks_raw) == 1 + + diff --git a/doajtest/unit/test_withdraw_reinstate.py b/doajtest/unit/test_withdraw_reinstate.py index 9c4464ceb0..425315b7a7 100644 --- a/doajtest/unit/test_withdraw_reinstate.py +++ b/doajtest/unit/test_withdraw_reinstate.py @@ -1,3 +1,4 @@ +from doajtest import helpers from doajtest.helpers import DoajTestCase from doajtest.fixtures import JournalFixtureFactory, ArticleFixtureFactory, ApplicationFixtureFactory from flask_login import current_user @@ -11,6 +12,7 @@ class TestWithdrawReinstate(DoajTestCase): def setUp(self): super(TestWithdrawReinstate, self).setUp() + helpers.initialise_index() def tearDown(self): super(TestWithdrawReinstate, self).tearDown() diff --git a/portality/api/current/crud/articles.py b/portality/api/current/crud/articles.py index c008e6b65a..fdcc4358dc 100644 --- a/portality/api/current/crud/articles.py +++ b/portality/api/current/crud/articles.py @@ -10,8 +10,10 @@ from portality.bll.doaj import DOAJ from portality.bll.exceptions import ArticleMergeConflict, ArticleNotAcceptable, DuplicateArticleException, \ IngestException +from portality.dao import ElasticSearchWriteException, DAOSaveExceptionMaxRetriesReached from copy import deepcopy + class ArticlesCrudApi(CrudApi): API_KEY_OPTIONAL = False @@ -91,6 +93,8 @@ def create(cls, data, account): raise Api403Error(str(e)) except IngestException as e: raise Api400Error(str(e)) + except (ElasticSearchWriteException, DAOSaveExceptionMaxRetriesReached) as e: + raise Api500Error(str(e)) # Check we are allowed to create an article for this journal @@ -241,6 +245,8 @@ def update(cls, id, data, account): raise Api400Error((str(e))) except DuplicateArticleException as e: raise Api403Error(str(e)) + except (ElasticSearchWriteException, DAOSaveExceptionMaxRetriesReached) as e: + raise Api500Error(str(e)) if result.get("success") == 0: raise Api400Error("Article update failed for unanticipated reason") diff --git a/portality/autocheck/__init__.py b/portality/autocheck/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/autocheck/checker.py b/portality/autocheck/checker.py new file mode 100644 index 0000000000..b229ecb3d4 --- /dev/null +++ b/portality/autocheck/checker.py @@ -0,0 +1,18 @@ +from portality.models import JournalLikeObject, Autocheck +from portality.autocheck.resource_bundle import ResourceBundle + +from typing import Callable + + +class Checker(object): + __identity__ = "base_checker" + + def name(self): + return self.__identity__ + + def check(self, form: dict, + jla: JournalLikeObject, + autochecks: Autocheck, + resources: ResourceBundle, + logger: Callable): + raise NotImplementedError() diff --git a/portality/autocheck/checkers/__init__.py b/portality/autocheck/checkers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/autocheck/checkers/issn_active.py b/portality/autocheck/checkers/issn_active.py new file mode 100644 index 0000000000..555da0718b --- /dev/null +++ b/portality/autocheck/checkers/issn_active.py @@ -0,0 +1,108 @@ +from portality.models import JournalLikeObject, Autocheck +from portality.autocheck.checker import Checker +from portality.autocheck.resource_bundle import ResourceBundle, ResourceUnavailable +from portality.autocheck.resources.issn_org import ISSNOrg +from typing import Callable + + +class ISSNChecker(Checker): + + UNABLE_TO_ACCESS = "unable_to_access" + + def retrieve_from_source(self, form, resources, autochecks, logger): + source = resources.resource(ISSNOrg) + + eissn = form.get("eissn") + pissn = form.get("pissn") + + eissn_data = None + pissn_data = None + eissn_url = None + pissn_url = None + eissn_fail = False + pissn_fail = False + + if eissn is not None: + eissn_url = source.reference_url(issn=eissn) + logger("Looking up eissn at {x}".format(x=eissn_url)) + try: + eissn_data = source.fetch(issn=eissn) + logger("Data received for eissn from {x}".format(x=eissn_url)) + except ResourceUnavailable: + logger("Unable to resolve eissn at {x}".format(x=eissn_url)) + autochecks.add_check( + field="eissn", + original_value=eissn, + advice=self.UNABLE_TO_ACCESS, + reference_url=eissn_url + ) + eissn_fail = True + + if pissn is not None: + pissn_url = source.reference_url(issn=pissn) + logger("Looking up pissn at {x}".format(x=pissn_url)) + try: + pissn_data = source.fetch(issn=pissn) + logger("Data received for pissn from {x}".format(x=pissn_url)) + except ResourceUnavailable: + logger("Unable to resolve pissn at {x}".format(x=pissn_url)) + autochecks.add_check( + field="pissn", + original_value=pissn, + advice=self.UNABLE_TO_ACCESS, + reference_url=pissn_url + ) + pissn_fail = True + + return eissn, eissn_url, eissn_data, eissn_fail, pissn, pissn_url, pissn_data, pissn_fail + + +class ISSNActive(ISSNChecker): + __identity__ = "issn_active" + + NOT_FOUND = "not_found" + FULLY_VALIDATED = "fully_validated" + NOT_VALIDATED = "not_validated" + + def _apply_rule(self, field, value, data, fail, url, logger, autochecks): + if value is not None: + if data is None: + if not fail: + logger("{y} not registered at {x}".format(y=field, x=url)) + autochecks.add_check( + field=field, + original_value=value, + advice=self.NOT_FOUND, + reference_url=url, + checked_by=self.__identity__ + ) + else: + if data.is_registered(): + logger("{y} confirmed as fully validated at {x}".format(y=value, x=url)) + autochecks.add_check( + field=field, + original_value=value, + advice=self.FULLY_VALIDATED, + reference_url=url, + checked_by=self.__identity__ + ) + else: + logger("{y} is not fully validated at {x}".format(y=value, x=url)) + autochecks.add_check( + field=field, + original_value=value, + advice=self.NOT_VALIDATED, + reference_url=url, + checked_by=self.__identity__ + ) + + def check(self, form: dict, + jla: JournalLikeObject, + autochecks: Autocheck, + resources: ResourceBundle, + logger: Callable): + + eissn, eissn_url, eissn_data, eissn_fail, pissn, pissn_url, pissn_data, pissn_fail = self.retrieve_from_source(form, resources, autochecks, logger) + + self._apply_rule("eissn", eissn, eissn_data, eissn_fail, eissn_url, logger, autochecks) + self._apply_rule("pissn", pissn, pissn_data, pissn_fail, pissn_url, logger, autochecks) \ No newline at end of file diff --git a/portality/autocheck/checkers/keepers_registry.py b/portality/autocheck/checkers/keepers_registry.py new file mode 100644 index 0000000000..c722c2d864 --- /dev/null +++ b/portality/autocheck/checkers/keepers_registry.py @@ -0,0 +1,108 @@ +from portality.models import JournalLikeObject, Autocheck +from portality.autocheck.resource_bundle import ResourceBundle +from typing import Callable +from portality.autocheck.checkers.issn_active import ISSNChecker +from datetime import datetime + + +class KeepersRegistry(ISSNChecker): + __identity__ = "keepers_registry" + + ID_MAP = { + "CLOCKSS": "http://issn.org/organization/keepers#clockss", + "LOCKSS": "http://issn.org/organization/keepers#lockss", + "Internet Archive": "http://issn.org/organization/keepers#internetarchive", + "PKP PN": "http://issn.org/organization/keepers#pkppln", + "Portico": "http://issn.org/organization/keepers#portico" + } + + MISSING = "missing" + PRESENT = "present" + OUTDATED = "outdated" + NOT_RECORDED = "not_recorded" + + def _get_archive_components(self, eissn_data, pissn_data): + acs = [] + if eissn_data is not None: + acs += eissn_data.archive_components + if pissn_data is not None: + acs += pissn_data.archive_components + return acs + + def _extract_archive_data(self, acs): + ad = {} + for ac in acs: + id = ac.get("holdingArchive", {}).get("@id") + tc = ac.get("temporalCoverage", "") + bits = tc.split("/") + if len(bits) != 2: + continue + end_year = int(bits[1].strip()) + if id in ad: + if end_year > ad[id]: + ad[id] = end_year + else: + ad[id] = end_year + + return ad + + def check(self, form: dict, + jla: JournalLikeObject, + autochecks: Autocheck, + resources: ResourceBundle, + logger: Callable): + + eissn, eissn_url, eissn_data, eissn_fail, pissn, pissn_url, pissn_data, pissn_fail = self.retrieve_from_source(form, resources, autochecks, logger) + + url = eissn_url if eissn_url else pissn_url + + acs = self._get_archive_components(eissn_data, pissn_data) + ad = self._extract_archive_data(acs) + services = form.get("preservation_service", []) + logger("There are {x} preservation services on the record: {y}".format(x=len(services), y=",".join(services))) + for service in services: + id = self.ID_MAP.get(service) + if not id: + logger("Service {x} is not recorded by Keepers Registry".format(x=service)) + autochecks.add_check( + field="preservation_service", + advice=self.NOT_RECORDED, + reference_url=url, + context={"service": service}, + checked_by=self.__identity__ + ) + continue + + coverage = ad.get(id) + if coverage is None: + # the archive is not mentioned in issn.org + logger("Service {x} is not registered at issn.org for this record".format(x=service)) + autochecks.add_check( + field="preservation_service", + advice=self.MISSING, + reference_url=url, + context={"service": service}, + checked_by=self.__identity__ + ) + continue + + if coverage < datetime.utcnow().year - 1: + # the temporal coverage is too old + logger("Service {x} is registerd as issn.org for this record, but the archive is not recent enough".format(x=service)) + autochecks.add_check( + field="preservation_service", + advice=self.OUTDATED, + reference_url=url, + context={"service": service}, + checked_by=self.__identity__ + ) + else: + # the coverage is within a reasonable period + logger("Service {x} is registerd as issn.org for this record".format(x=service)) + autochecks.add_check( + field="preservation_service", + advice=self.PRESENT, + reference_url=url, + context={"service": service}, + checked_by=self.__identity__ + ) \ No newline at end of file diff --git a/portality/autocheck/resource_bundle.py b/portality/autocheck/resource_bundle.py new file mode 100644 index 0000000000..1355eb46f6 --- /dev/null +++ b/portality/autocheck/resource_bundle.py @@ -0,0 +1,63 @@ +class ResourceUnavailable(Exception): + pass + + +class ResourceBundle(object): + def __init__(self, resources=None, stack_size=100): + self._resources = resources if resources is not None else [] + self._resource_data = {} + self._resource_stack = [] + self._stack_size = stack_size + + def register(self, resource_id, data): + self._resource_data[resource_id] = data + self._resource_stack.append(resource_id) + if len(self._resource_stack) > self._stack_size: + remove = self._resource_stack.pop(0) + if remove in self._resource_data: + # I don't know why it ever wouldn't be, but I have occasionally got a key error here + self._resource_data.pop(remove) + + def get(self, resource_id): + return self._resource_data.get(resource_id) + + def resource(self, resource_class): + for resource in self._resources: + if isinstance(resource, resource_class): + return resource + inst = resource_class(self) + self._resources.append(inst) + return inst + + +class Resource(object): + __identity__ = "base_resource" + + def __init__(self, resource_bundle): + self._resource_bundle = resource_bundle + + def name(self): + return self.__identity__ + + def make_resource_id(self, *args, **kwargs): + return self.name() + + def reference_url(self, *args, **kwargs): + raise NotImplementedError() + + def fetch_fresh(self, *args, **kwargs): + raise NotImplementedError() + + def fetch(self, *args, **kwargs): + resource_id = self.make_resource_id(*args, **kwargs) + data = self._resource_bundle.get(resource_id) + if data is not None: + return data + + try: + data = self.fetch_fresh(*args, **kwargs) + except Exception as e: + raise ResourceUnavailable() + + self._resource_bundle.register(resource_id, data) + return data \ No newline at end of file diff --git a/portality/autocheck/resources/__init__.py b/portality/autocheck/resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/autocheck/resources/issn_org.py b/portality/autocheck/resources/issn_org.py new file mode 100644 index 0000000000..febe3e7291 --- /dev/null +++ b/portality/autocheck/resources/issn_org.py @@ -0,0 +1,61 @@ +from datetime import datetime + +from portality.autocheck.resource_bundle import Resource +from portality.core import app + +import requests +import json +import time +from bs4 import BeautifulSoup + + +class ISSNOrg(Resource): + __identity__ = "issn_org" + + def __init__(self, resource_bundle): + super(ISSNOrg, self).__init__(resource_bundle) + self._timeout = app.config.get("AUTOCHECK_RESOURCE_ISSN_ORG_TIMEOUT", 10) + self._throttle = app.config.get("AUTOCHECK_RESOURCE_ISSN_ORG_THROTTLE", 0) + self._last_request = None + + def make_resource_id(self, issn): + return self.name() + "_" + issn + + def reference_url(self, issn): + return "https://portal.issn.org/resource/ISSN/" + issn + + def fetch_fresh(self, issn): + if self._last_request is not None: + now = datetime.utcnow() + since_last = (now - self._last_request).total_seconds() + if since_last < self._throttle: + time.sleep(self._throttle - since_last) + + resp = requests.get(self.reference_url(issn), timeout=self._timeout) + self._last_request = datetime.utcnow() + + page = BeautifulSoup(resp.text, features="lxml") + + scripts = page.find_all("script", type="application/ld+json") + if len(scripts) == 0: + return None + + raw = scripts[0].string + data = json.loads(raw) + return ISSNOrgData(data) + + +class ISSNOrgData(object): + def __init__(self, raw): + self.data = raw + + @property + def version(self): + return self.data.get("mainEntityOfPage", {}).get("version") + + def is_registered(self): + return self.version == "Register" + + @property + def archive_components(self): + return [ac for ac in self.data.get("subjectOf", []) if ac.get("@type") == "ArchiveComponent"] diff --git a/portality/bll/doaj.py b/portality/bll/doaj.py index 584daa9b8f..bd756e8b59 100644 --- a/portality/bll/doaj.py +++ b/portality/bll/doaj.py @@ -126,3 +126,8 @@ def tourService(cls): """ from portality.bll.services import tour return tour.TourService() + + @classmethod + def autochecksService(cls, autocheck_plugins=None): + from portality.bll.services import autochecks + return autochecks.AutocheckService(autocheck_plugins=autocheck_plugins) \ No newline at end of file diff --git a/portality/bll/services/authorisation.py b/portality/bll/services/authorisation.py index 75d0a2b607..5be4e65c77 100644 --- a/portality/bll/services/authorisation.py +++ b/portality/bll/services/authorisation.py @@ -55,7 +55,7 @@ def can_edit_application(self, account, application): if account.has_role("publisher"): if account.id != application.owner: no_auth_reason = exceptions.AuthoriseException.NOT_OWNER - elif application.application_status not in [ + elif application.application_status not in [ # ~~-> ApplicationStatuses:Config~~ constants.APPLICATION_STATUS_PENDING, constants.APPLICATION_STATUS_UPDATE_REQUEST, constants.APPLICATION_STATUS_REVISIONS_REQUIRED diff --git a/portality/bll/services/autochecks.py b/portality/bll/services/autochecks.py new file mode 100644 index 0000000000..0d00fe512c --- /dev/null +++ b/portality/bll/services/autochecks.py @@ -0,0 +1,122 @@ +from portality.crosswalks.application_form import ApplicationFormXWalk, JournalFormXWalk +from portality.autocheck.resource_bundle import ResourceBundle +from portality import models + +from portality.autocheck.checkers.issn_active import ISSNActive +from portality.autocheck.checkers.keepers_registry import KeepersRegistry + +AUTOCHECK_PLUGINS = [ + # (Active on Journal?, Active on Application?, Plugin Class) + (True, True, ISSNActive), + (True, True, KeepersRegistry) +] + + +class AutocheckService(object): + """ + ~~Autochecks:Service->DOAJ:Service~~ + """ + + def __init__(self, autocheck_plugins=None): + self._autocheck_plugins = autocheck_plugins if autocheck_plugins is not None else AUTOCHECK_PLUGINS + + def autocheck_applications(self, application_ids=None, logger=None): + """ + ~~Autochecks:Service->DOAJ:Service~~ + """ + resource_bundle = ResourceBundle() + if application_ids is None: + for application in models.Application.iterate(): + self.autocheck_application(application, logger=logger, resource_bundle=resource_bundle) + else: + for application_id in application_ids: + application = models.Application.pull(application_id) + if application is None: + continue + self.autocheck_application(application, logger=logger, resource_bundle=resource_bundle) + + def autocheck_application(self, application: models.Application, created_date=None, logger=None, resource_bundle=None): + if logger is None: + logger = lambda x: x # does nothing, just swallows the logs + + if resource_bundle is None: + resource_bundle = ResourceBundle() + + application_form = ApplicationFormXWalk.obj2form(application) + new_autochecks = models.Autocheck() + new_autochecks.application = application.id + + if created_date is not None: + new_autochecks.set_created(created_date) + + for j, a, klazz in self._autocheck_plugins: + if not a: + continue + + checker = klazz() + logger("Running autocheck plugin {x}".format(x=checker.name())) + checker.check(application_form, application, new_autochecks, resource_bundle, logger) + + new_autochecks.save() + logger("Saved new autocheck document {id}".format(id=new_autochecks.id)) + + models.Autocheck.delete_all_but_latest(application_id=application.id) + + return new_autochecks + + def autocheck_journals(self, journal_ids=None, logger=None): + """ + ~~autochecks:Service->DOAJ:Service~~ + """ + resource_bundle = ResourceBundle() + if journal_ids is None: + for journal in models.Journal.iterate(): + self.autocheck_journal(journal, logger=logger, resource_bundle=resource_bundle) + else: + for journal_id in journal_ids: + journal = models.Journal.pull(journal_id) + if journal is None: + continue + self.autocheck_journal(journal, logger=logger, resource_bundle=resource_bundle) + + def autocheck_journal(self, journal: models.Journal, logger=None, resource_bundle=None): + if logger is None: + logger = lambda x: x # does nothing, just swallows the logs + + if resource_bundle is None: + resource_bundle = ResourceBundle() + + journal_form = JournalFormXWalk.obj2form(journal) + new_autochecks = models.Autocheck() + new_autochecks.journal = journal.id + + for j, a, klazz in self._autocheck_plugins: + if not j: + continue + + checker = klazz() + logger("Running autocheck plugin {x}".format(x=checker.name())) + checker.check(journal_form, journal, new_autochecks, resource_bundle, logger) + + new_autochecks.save() + logger("Saved new autocheck document {id}".format(id=new_autochecks.id)) + + models.Autocheck.delete_all_but_latest(journal_id=journal.id) + + return new_autochecks + + def dismiss(self, autocheck_set_id, autocheck_id): + autochecks = models.Autocheck.pull(autocheck_set_id) + if autochecks is None: + return False + autochecks.dismiss(autocheck_id) + autochecks.save() + return True + + def undismiss(self, autocheck_set_id, autocheck_id): + autochecks = models.Autocheck.pull(autocheck_set_id) + if autochecks is None: + return False + autochecks.undismiss(autocheck_id) + autochecks.save() + return True \ No newline at end of file diff --git a/portality/bll/services/site.py b/portality/bll/services/site.py index f7dfa45d02..a2a4078381 100644 --- a/portality/bll/services/site.py +++ b/portality/bll/services/site.py @@ -79,12 +79,14 @@ def sitemap(self, prune: bool = True): create_url_element(urlset, u, toc_changefreq) counter += 1 - # do all the journal ToCs + # do all the journal ToCs and articles for j in models.Journal.all_in_doaj(): # first create an entry purely for the journal toc_loc = base_url + "toc/" + j.toc_id + toc_art_loc = base_url + "toc/" + j.toc_id + "/articles" create_url_element(urlset, toc_loc, toc_changefreq, lastmod=j.last_updated) - counter += 1 + create_url_element(urlset, toc_art_loc, toc_changefreq) + counter += 2 # log to the screen action_register.append("{x} urls written to sitemap".format(x=counter)) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 74c49645fa..fc57f66da7 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -2,10 +2,13 @@ from portality import models from portality.bll import exceptions from portality import constants +from portality.lib import dates +from datetime import datetime class TodoService(object): """ ~~Todo:Service->DOAJ:Service~~ + ~~-> ApplicationStatuses:Config~~ """ def group_stats(self, group_id): @@ -61,7 +64,7 @@ def group_stats(self, group_id): return stats - def top_todo(self, account, size=25): + def top_todo(self, account, size=25, new_applications=True, update_requests=True): """ Returns the top number of todo items for a given user @@ -77,39 +80,44 @@ def top_todo(self, account, size=25): queries = [] if account.has_role("admin"): maned_of = models.EditorGroup.groups_by_maned(account.id) - queries.append(TodoRules.maned_follow_up_old(size, maned_of)) - queries.append(TodoRules.maned_stalled(size, maned_of)) - queries.append(TodoRules.maned_ready(size, maned_of)) - queries.append(TodoRules.maned_completed(size, maned_of)) - queries.append(TodoRules.maned_assign_pending(size, maned_of)) - - if account.has_role("editor"): - groups = [g for g in models.EditorGroup.groups_by_editor(account.id)] - regular_groups = [g for g in groups if g.maned != account.id] - maned_groups = [g for g in groups if g.maned == account.id] - if len(groups) > 0: - queries.append(TodoRules.editor_follow_up_old(groups, size)) - queries.append(TodoRules.editor_stalled(groups, size)) - queries.append(TodoRules.editor_completed(groups, size)) - - # for groups where the user is not the maned for a group, given them the assign - # pending todos at the regular priority - if len(regular_groups) > 0: - queries.append(TodoRules.editor_assign_pending(regular_groups, size)) - - # for groups where the user IS the maned for a group, give them the assign - # pending todos at a lower priority - if len(maned_groups) > 0: - qi = TodoRules.editor_assign_pending(maned_groups, size) - queries.append((constants.TODO_EDITOR_ASSIGN_PENDING_LOW_PRIORITY, qi[1], qi[2], -2)) - - if account.has_role(constants.ROLE_ASSOCIATE_EDITOR): - queries.extend([ - TodoRules.associate_follow_up_old(account.id, size), - TodoRules.associate_stalled(account.id, size), - TodoRules.associate_start_pending(account.id, size), - TodoRules.associate_all_applications(account.id, size) - ]) + if new_applications: + queries.append(TodoRules.maned_follow_up_old(size, maned_of)) + queries.append(TodoRules.maned_stalled(size, maned_of)) + queries.append(TodoRules.maned_ready(size, maned_of)) + queries.append(TodoRules.maned_completed(size, maned_of)) + queries.append(TodoRules.maned_assign_pending(size, maned_of)) + if update_requests: + queries.append(TodoRules.maned_last_month_update_requests(size, maned_of)) + queries.append(TodoRules.maned_new_update_requests(size, maned_of)) + + if new_applications: # editor and associate editor roles only deal with new applications + if account.has_role("editor"): + groups = [g for g in models.EditorGroup.groups_by_editor(account.id)] + regular_groups = [g for g in groups if g.maned != account.id] + maned_groups = [g for g in groups if g.maned == account.id] + if len(groups) > 0: + queries.append(TodoRules.editor_follow_up_old(groups, size)) + queries.append(TodoRules.editor_stalled(groups, size)) + queries.append(TodoRules.editor_completed(groups, size)) + + # for groups where the user is not the maned for a group, given them the assign + # pending todos at the regular priority + if len(regular_groups) > 0: + queries.append(TodoRules.editor_assign_pending(regular_groups, size)) + + # for groups where the user IS the maned for a group, give them the assign + # pending todos at a lower priority + if len(maned_groups) > 0: + qi = TodoRules.editor_assign_pending(maned_groups, size) + queries.append((constants.TODO_EDITOR_ASSIGN_PENDING_LOW_PRIORITY, qi[1], qi[2], -2)) + + if account.has_role(constants.ROLE_ASSOCIATE_EDITOR): + queries.extend([ + TodoRules.associate_follow_up_old(account.id, size), + TodoRules.associate_stalled(account.id, size), + TodoRules.associate_start_pending(account.id, size), + TodoRules.associate_all_applications(account.id, size) + ]) todos = [] for aid, q, sort, boost in queries: @@ -162,7 +170,8 @@ def maned_stalled(cls, size, maned_of): stalled = TodoQuery( musts=[ TodoQuery.lmu_older_than(8), - TodoQuery.editor_group(maned_of) + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application() ], must_nots=[ TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) @@ -178,7 +187,8 @@ def maned_follow_up_old(cls, size, maned_of): follow_up_old = TodoQuery( musts=[ TodoQuery.cd_older_than(10), - TodoQuery.editor_group(maned_of) + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application() ], must_nots=[ TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) @@ -194,7 +204,8 @@ def maned_ready(cls, size, maned_of): ready = TodoQuery( musts=[ TodoQuery.status([constants.APPLICATION_STATUS_READY]), - TodoQuery.editor_group(maned_of) + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application() ], sort=sort_date, size=size @@ -208,7 +219,8 @@ def maned_completed(cls, size, maned_of): musts=[ TodoQuery.status([constants.APPLICATION_STATUS_COMPLETED]), TodoQuery.lmu_older_than(2), - TodoQuery.editor_group(maned_of) + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application() ], sort=sort_date, size=size @@ -223,7 +235,8 @@ def maned_assign_pending(cls, size, maned_of): TodoQuery.exists("admin.editor_group"), TodoQuery.lmu_older_than(2), TodoQuery.status([constants.APPLICATION_STATUS_PENDING]), - TodoQuery.editor_group(maned_of) + TodoQuery.editor_group(maned_of), + TodoQuery.is_new_application() ], must_nots=[ TodoQuery.exists("admin.editor") @@ -233,6 +246,50 @@ def maned_assign_pending(cls, size, maned_of): ) return constants.TODO_MANED_ASSIGN_PENDING, assign_pending, sort_date, 0 + @classmethod + def maned_last_month_update_requests(cls, size, maned_of): + som = dates.now().replace(day=1, hour=0, minute=0, second=0, microsecond=0) + now = dates.now() + since_som = int((now - som).total_seconds()) + + sort_date = "created_date" + assign_pending = TodoQuery( + musts=[ + TodoQuery.exists("admin.editor_group"), + TodoQuery.cd_older_than(since_som, unit="s"), + # TodoQuery.status([constants.APPLICATION_STATUS_UPDATE_REQUEST]), + TodoQuery.editor_group(maned_of), + TodoQuery.is_update_request() + ], + must_nots=[ + TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + # TodoQuery.exists("admin.editor") + ], + sort=sort_date, + size=size + ) + return constants.TODO_MANED_LAST_MONTH_UPDATE_REQUEST, assign_pending, sort_date, 2 + + @classmethod + def maned_new_update_requests(cls, size, maned_of): + sort_date = "created_date" + assign_pending = TodoQuery( + musts=[ + TodoQuery.exists("admin.editor_group"), + # TodoQuery.cd_older_than(4), + # TodoQuery.status([constants.APPLICATION_STATUS_UPDATE_REQUEST]), + TodoQuery.editor_group(maned_of), + TodoQuery.is_update_request() + ], + must_nots=[ + TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED]) + # TodoQuery.exists("admin.editor") + ], + sort=sort_date, + size=size + ) + return constants.TODO_MANED_NEW_UPDATE_REQUEST, assign_pending, sort_date, -2 + @classmethod def editor_stalled(cls, groups, size): sort_date = "created_date" @@ -427,6 +484,14 @@ def is_new_application(cls): } } + @classmethod + def is_update_request(cls): + return { + "term": { + "admin.application_type.exact": constants.APPLICATION_TYPE_UPDATE_REQUEST + } + } + @classmethod def editor_group(cls, groups): return { @@ -446,11 +511,11 @@ def lmu_older_than(cls, weeks): } @classmethod - def cd_older_than(cls, weeks): + def cd_older_than(cls, count, unit="w"): return { "range": { "admin.date_applied": { - "lte": "now-" + str(weeks) + "w" + "lte": "now-" + str(count) + unit } } } diff --git a/portality/bll/services/tour.py b/portality/bll/services/tour.py index 31dae4767b..917cc8e088 100644 --- a/portality/bll/services/tour.py +++ b/portality/bll/services/tour.py @@ -5,7 +5,7 @@ def activeTours(self, path, user): tours = app.config.get("TOURS", {}) active_tours = [] for k, v in tours.items(): - if path == k: + if self._is_active_path(path, k): for tour in v: if "roles" in tour: if user is None: @@ -18,6 +18,20 @@ def activeTours(self, path, user): active_tours.append(tour) return active_tours + def _is_active_path(self, given, candidate): + sw = False + if candidate.endswith("*"): + candidate = candidate[:-1] + sw = True + + if not sw and given == candidate: + return True + + if sw and given.startswith(candidate): + return True + + return False + def validateContentId(self, content_id): tours = app.config.get("TOURS", {}) for k, v in tours.items(): diff --git a/portality/constants.py b/portality/constants.py index 08629f0e1d..7d0eda81e6 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -1,4 +1,6 @@ # ~~Constants:Config~~ + +# ~~-> ApplicationStatuses:Config~~ APPLICATION_STATUS_ACCEPTED = "accepted" APPLICATION_STATUS_REJECTED = "rejected" APPLICATION_STATUS_UPDATE_REQUEST = "update_request" @@ -8,6 +10,7 @@ APPLICATION_STATUS_COMPLETED = "completed" APPLICATION_STATUS_ON_HOLD = "on hold" APPLICATION_STATUS_READY = "ready" +APPLICATION_STATUS_POST_SUBMISSION_REVIEW = "post_submission_review" APPLICATION_STATUSES_ALL = [ APPLICATION_STATUS_ACCEPTED, @@ -18,7 +21,8 @@ APPLICATION_STATUS_IN_PROGRESS, APPLICATION_STATUS_COMPLETED, APPLICATION_STATUS_ON_HOLD, - APPLICATION_STATUS_READY + APPLICATION_STATUS_READY, + APPLICATION_STATUS_POST_SUBMISSION_REVIEW ] APPLICATION_TYPE_UPDATE_REQUEST = "update_request" @@ -44,6 +48,8 @@ TODO_MANED_READY = "todo_maned_ready" TODO_MANED_COMPLETED = "todo_maned_completed" TODO_MANED_ASSIGN_PENDING = "todo_maned_assign_pending" +TODO_MANED_LAST_MONTH_UPDATE_REQUEST = "todo_maned_last_month_update_request" +TODO_MANED_NEW_UPDATE_REQUEST = "todo_maned_new_update_request" TODO_EDITOR_STALLED = "todo_editor_stalled" TODO_EDITOR_FOLLOW_UP_OLD = "todo_editor_follow_up_old" TODO_EDITOR_COMPLETED = "todo_editor_completed" @@ -78,6 +84,8 @@ PROCESS__QUICK_REJECT = "quick_reject" # Role +ROLE_ADMIN = "admin" +ROLE_PUBLISHER = "publisher" ROLE_ASSOCIATE_EDITOR = 'associate_editor' ROLE_PUBLIC_DATA_DUMP = "public_data_dump" ROLE_PUBLISHER = "publisher" diff --git a/portality/core.py b/portality/core.py index c74493a9a0..2eb486ea8e 100644 --- a/portality/core.py +++ b/portality/core.py @@ -213,7 +213,8 @@ def put_mappings(conn, mappings): for key, mapping in iter(mappings.items()): altered_key = app.config['ELASTIC_SEARCH_DB_PREFIX'] + key if not conn.indices.exists(altered_key): - r = conn.indices.create(index=altered_key, body=mapping) + r = conn.indices.create(index=altered_key, body=mapping, + request_timeout=app.config.get("ES_SOCKET_TIMEOUT", None)) print("Creating ES Type + Mapping in index {0} for {1}; status: {2}".format(altered_key, key, r)) else: print("ES Type + Mapping already exists in index {0} for {1}".format(altered_key, key)) diff --git a/portality/crosswalks/journal_questions.py b/portality/crosswalks/journal_questions.py index 65e9e6729d..1a8d12b0e6 100644 --- a/portality/crosswalks/journal_questions.py +++ b/portality/crosswalks/journal_questions.py @@ -34,7 +34,7 @@ class Journal2QuestionXwalk(object): ("eissn", "Journal EISSN (online version)"), ("continues", "Continues"), ("continued_by", "Continued By"), - ("institution_name", "Society or institution"), + ("institution_name", "Other organisation"), ("keywords", "Keywords"), ("language", "Languages in which the journal accepts manuscripts"), ("license_attributes", "License attributes"), @@ -57,7 +57,7 @@ class Journal2QuestionXwalk(object): ("publisher_name", "Publisher"), ("other_charges_url", "Other fees information URL"), ("title", "Journal title"), - ("institution_country", "Country of society or institution"), + ("institution_country", "Country of other organisation"), ("apc", "APC"), ("has_other_charges", "Has other fees"), ("has_waiver", "Journal waiver policy (for developing country authors etc)"), @@ -334,7 +334,7 @@ def csv2formval(key, value): continue # Only deal with a question if there's a value - TODO: what does this mean for yes_no_or_blank? - if len(v.strip()) > 0: + if isinstance(v, str) and len(v.strip()) > 0: # Get the question key from the CSV column header form_key = cls.p(k) diff --git a/portality/dao.py b/portality/dao.py index 4688662338..1c2b32da5f 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -51,8 +51,7 @@ class DomainObject(UserDict, object): which provide interaction with ES index such as (save, delete, query, etc.) """ - # set the type on the model that inherits this - # which also is ES index name of the model + # set the type on the model that inherits this# which also is ES index name of the model __type__ = None def __init__(self, **kwargs): @@ -178,7 +177,7 @@ def save(self, retries=0, back_off_factor=1, differentiate=False, blocking=False r = ES.index(self.index_name(), d, doc_type=self.doc_type(), id=self.data.get("id"), headers=CONTENT_TYPE_JSON, - timeout=app.config.get('ES_READ_TIMEOUT', '1m'), ) + timeout=app.config.get('ES_READ_TIMEOUT', None), ) break except (elasticsearch.ConnectionError, elasticsearch.ConnectionTimeout): @@ -442,7 +441,7 @@ def send_query(cls, qobj, retry=50, **kwargs): # ES 7.10 updated target to whole index, since specifying type for search is deprecated # r = requests.post(cls.target_whole_index() + recid + "_search", data=json.dumps(qobj), headers=CONTENT_TYPE_JSON) if kwargs.get('timeout') is None: - kwargs['timeout'] = app.config.get('ES_READ_TIMEOUT', '1m') + kwargs['timeout'] = app.config.get('ES_READ_TIMEOUT', None) r = ES.search(body=json.dumps(qobj), index=cls.index_name(), doc_type=cls.doc_type(), headers=CONTENT_TYPE_JSON, **kwargs) break @@ -890,9 +889,10 @@ def block(cls, id, last_updated=None, sleep=0.5, max_retry_seconds=30): return else: if (dates.now() - start_time).total_seconds() >= max_retry_seconds: - raise BlockTimeOutException( - "Attempting to block until record with id {id} appears in Elasticsearch, but this has not happened after {limit}".format( - id=id, limit=max_retry_seconds)) + raise (BlockTimeOutException( + "Attempting to block until record with id {id} appears in Elasticsearch, but this has not happened after {limit}" + .format( + id=id, limit=max_retry_seconds))) time.sleep(sleep) @@ -934,8 +934,30 @@ def save_all(cls, models, blocking=False): cls.blockall((m.id, getattr(m, "last_updated", None)) for m in models) +def any_pending_tasks(): + """ Check if there are any pending tasks in the elasticsearch task queue """ + results = ES.cluster.pending_tasks() + return len(results["tasks"]) > 0 + + +def query_data_tasks(timeout='30s'): + """ Check if there are any pending tasks in the elasticsearch task queue """ + results = ES.tasks.list(params={ + "actions": 'indices:data*', + "timeout": timeout, + "wait_for_completion": 'true', + }) + tasks = [] + for node in results['nodes'].values(): + tasks.extend(node['tasks'].values()) + return tasks +def refresh(): + """ + refresh all indexes to make newly added or deleted documents immediately searchable + """ + return ES.indices.refresh() class BlockTimeOutException(Exception): @@ -958,7 +980,6 @@ class ESError(Exception): pass - ######################################################################## # Some useful ES queries ######################################################################## @@ -1013,7 +1034,6 @@ def query(self): } - class WildcardAutocompleteQuery(object): def __init__(self, wildcard_field, wildcard_query, agg_name, agg_field, agg_size): self._wildcard_field = wildcard_field diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py index 936e6f0482..3ac5f845b9 100644 --- a/portality/forms/application_forms.py +++ b/portality/forms/application_forms.py @@ -44,6 +44,7 @@ from portality.lib.formulaic import Formulaic, WTFormsBuilder, FormulaicContext, FormulaicField from portality.models import EditorGroup from portality.regex import ISSN, ISSN_COMPILED +from portality.ui.messages import Messages # Stop words used in the keywords field STOP_WORDS = [ @@ -305,7 +306,12 @@ class FieldDefinitions: "The ISSN must match what is given on the journal website."], "placeholder": "", "doaj_criteria": "ISSN must be provided" - } + }, + "widgets": [ + "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ + "autocheck", # ~~^-> Autocheck:FormWidget~~ + "issn_link" # ~~^->IssnLink:FormWidget~~ + ] }, "editor": { "disabled": True, @@ -354,7 +360,6 @@ class FieldDefinitions: ], "widgets" : [ "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ - "full_contents", # ~~^->FullContents:FormWidget~~ "issn_link" # ~~^->IssnLink:FormWidget~~ ], "contexts": { @@ -375,7 +380,12 @@ class FieldDefinitions: "The ISSN must match what is given on the journal website."], "placeholder": "", "doaj_criteria": "ISSN must be provided" - } + }, + "widgets": [ + "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ + "autocheck", # ~~^-> Autocheck:FormWidget~~ + "issn_link" # ~~^->IssnLink:FormWidget~~ + ] }, "editor": { "disabled": True, @@ -426,9 +436,6 @@ class FieldDefinitions: {"stop_words": {"disallowed": STOP_WORDS}}, # ~~^->StopWords:FormValidator~~ {"max_tags": {"max": 6}} ], - "postprocessing": [ - "to_lower" # FIXME: this might just be a feature of the crosswalk - ], "widgets": [ { "taglist": { @@ -1014,9 +1021,6 @@ class FieldDefinitions: {"required": {"message": "Enter an average number of weeks"}}, {"int_range": {"gte": 1, "lte": 100}} ], - "asynchronous_warning": [ - {"int_range": {"lte": 2}} - ], "attr": { "min": "1", "max": "100" @@ -1277,7 +1281,14 @@ class FieldDefinitions: }, "validate": [ {"required": {"message": "Select at least one option"}} - ] + ], + "contexts" : { + "admin": { + "widgets": [ + "autocheck", # ~~^-> Autocheck:FormWidget~~ + ] + } + } } # ~~->$ PreservationServiceLibrary:FormField~~ @@ -1301,9 +1312,6 @@ class FieldDefinitions: } } ], - "asynchronous_warning": [ - {"warn_on_value": {"value": "None"}} - ], "widgets": [ "trim_whitespace", # ~~^-> TrimWhitespace:FormWidget~~ "multiple_field" @@ -1327,9 +1335,6 @@ class FieldDefinitions: } } ], - "asynchronous_warning": [ - {"warn_on_value": {"value": "None"}} - ], "widgets" : [ "trim_whitespace" # ~~^-> TrimWhitespace:FormWidget~~ ] @@ -1390,9 +1395,10 @@ class FieldDefinitions: "input": "checkbox", "multiple": True, "options": [ - {"display": "Sherpa/Romeo", "value": "Sherpa/Romeo", "subfields": ["deposit_policy_url"]}, - {"display": "Dulcinea", "value": "Dulcinea", "subfields": ["deposit_policy_url"]}, {"display": "Diadorim", "value": "Diadorim", "subfields": ["deposit_policy_url"]}, + {"display": "Dulcinea", "value": "Dulcinea", "subfields": ["deposit_policy_url"]}, + {"display": "Mir@bel", "value": "Mir@bel", "subfields": ["deposit_policy_url"]}, + {"display": "Sherpa/Romeo", "value": "Sherpa/Romeo", "subfields": ["deposit_policy_url"]}, {"display": "Other (including publisher’s own site)", "value": "other", "subfields": ["deposit_policy_other", "deposit_policy_url"]}, {"display": "The journal has no repository policy", "value": "none", "exclusive": True} ], @@ -1426,9 +1432,6 @@ class FieldDefinitions: } } ], - "asynchronous_warning": [ - {"warn_on_value": {"value": "None"}} - ], "widgets" : [ "trim_whitespace" # ~~^-> TrimWhitespace:FormWidget~~ ] @@ -1440,10 +1443,10 @@ class FieldDefinitions: "label": "Where can we find this information?", "input": "text", "diff_table_context": "Repository policy", - "conditional": [{"field": "deposit_policy", "value": "Sherpa/Romeo"}, + "conditional": [{"field": "deposit_policy", "value": "Diadorim"}, {"field": "deposit_policy", "value": "Dulcinea"}, - {"field": "deposit_policy", "value": "Diadorim"}, - {"field": "deposit_policy", "value": "Diadorim"}, + {"field": "deposit_policy", "value": "Mir@bel"}, + {"field": "deposit_policy", "value": "Sherpa/Romeo"}, {"field": "deposit_policy", "value": "other"}], "help": { "doaj_criteria": "You must provide a URL", @@ -1464,9 +1467,10 @@ class FieldDefinitions: "required_if": { "field": "deposit_policy", "value": [ - "Sherpa/Romeo", - "Dulcinea", "Diadorim", + "Dulcinea", + "Mir@bel", + "Sherpa/Romeo", "other" ] } @@ -1480,9 +1484,10 @@ class FieldDefinitions: "required_if": { "field": "deposit_policy", "value": [ - "Sherpa/Romeo", - "Dulcinea", "Diadorim", + "Dulcinea", + "Mir@bel", + "Sherpa/Romeo", "other" ] } @@ -1532,9 +1537,6 @@ class FieldDefinitions: } } ], - "asynchronous_warning": [ - {"warn_on_value": {"value": "None"}} - ], "widgets" : [ "trim_whitespace" # ~~^-> TrimWhitespace:FormWidget~~ ] @@ -1596,12 +1598,12 @@ class FieldDefinitions: } ####################################### - ## Ediorial fields + ## Editorial fields # ~~->$ DOAJSeal:FormField~~ DOAJ_SEAL = { "name": "doaj_seal", - "label": "The journal has fulfilled all the criteria for the Seal. Award the Seal?", + "label": "The journal may have fulfilled all the criteria for the Seal. Award the Seal?", "input": "checkbox", "validate": [ { @@ -2506,24 +2508,30 @@ def quick_reject(field, formulaic_context_name): def application_statuses(field, formulaic_context): # ~~->$ ApplicationStatus:Workflow~~ + # ~~-> ApplicationStatuses:Config~~ _application_status_base = [ # This is all the Associate Editor sees ('', ' '), - (constants.APPLICATION_STATUS_PENDING, 'Pending'), - (constants.APPLICATION_STATUS_IN_PROGRESS, 'In Progress'), - (constants.APPLICATION_STATUS_COMPLETED, 'Completed') + (constants.APPLICATION_STATUS_PENDING, Messages.FORMS__APPLICATION_STATUS__PENDING), + (constants.APPLICATION_STATUS_IN_PROGRESS, Messages.FORMS__APPLICATION_STATUS__IN_PROGRESS), + (constants.APPLICATION_STATUS_COMPLETED, Messages.FORMS__APPLICATION_STATUS__COMPLETED) ] + # Note that an admin is given the Post Submission Automation status, as technically they + # may edit an item that's in this status, but it is functionally useless to them + # It would be nice to be able to somehow disable it being changed, perhaps we can do that + # via a widget _application_status_admin = _application_status_base + [ - (constants.APPLICATION_STATUS_UPDATE_REQUEST, 'Update Request'), - (constants.APPLICATION_STATUS_REVISIONS_REQUIRED, 'Revisions Required'), - (constants.APPLICATION_STATUS_ON_HOLD, 'On Hold'), - (constants.APPLICATION_STATUS_READY, 'Ready'), - (constants.APPLICATION_STATUS_REJECTED, 'Rejected'), - (constants.APPLICATION_STATUS_ACCEPTED, 'Accepted') + (constants.APPLICATION_STATUS_POST_SUBMISSION_REVIEW, Messages.FORMS__APPLICATION_STATUS__POST_SUBMISSION_REVIEW), + (constants.APPLICATION_STATUS_UPDATE_REQUEST, Messages.FORMS__APPLICATION_STATUS__UPDATE_REQUEST), + (constants.APPLICATION_STATUS_REVISIONS_REQUIRED, Messages.FORMS__APPLICATION_STATUS__REVISIONS_REQUIRED), + (constants.APPLICATION_STATUS_ON_HOLD, Messages.FORMS__APPLICATION_STATUS__ON_HOLD), + (constants.APPLICATION_STATUS_READY, Messages.FORMS__APPLICATION_STATUS__READY), + (constants.APPLICATION_STATUS_REJECTED, Messages.FORMS__APPLICATION_STATUS__REJECTED), + (constants.APPLICATION_STATUS_ACCEPTED, Messages.FORMS__APPLICATION_STATUS__ACCEPTED) ] _application_status_editor = _application_status_base + [ - (constants.APPLICATION_STATUS_READY, 'Ready'), + (constants.APPLICATION_STATUS_READY, Messages.FORMS__APPLICATION_STATUS__READY), ] formulaic_context_name = None @@ -2536,7 +2544,7 @@ def application_statuses(field, formulaic_context): elif formulaic_context_name == "editor": status_list = _application_status_editor elif formulaic_context_name == "accepted": - status_list = [(constants.APPLICATION_STATUS_ACCEPTED, 'Accepted')] # just the one status - Accepted + status_list = [(constants.APPLICATION_STATUS_ACCEPTED, Messages.FORMS__APPLICATION_STATUS__ACCEPTED)] # just the one status - Accepted else: status_list = _application_status_base @@ -3032,8 +3040,9 @@ def wtforms(field, settings): "full_contents" : "formulaic.widgets.newFullContents", # ~~^->FullContents:FormWidget~~ "load_editors" : "formulaic.widgets.newLoadEditors", # ~~-> LoadEditors:FormWidget~~ "trim_whitespace" : "formulaic.widgets.newTrimWhitespace", # ~~-> TrimWhitespace:FormWidget~~ - "note_modal" : "formulaic.widgets.newNoteModal", # ~~-> NoteModal:FormWidget~~, - "issn_link" : "formulaic.widgets.newIssnLink" # ~~-> IssnLink:FormWidget~~, + "note_modal" : "formulaic.widgets.newNoteModal", # ~~-> NoteModal:FormWidget~~ + "autocheck": "formulaic.widgets.newAutocheck", # ~~-> Autocheck:FormWidget~~ + "issn_link" : "formulaic.widgets.newIssnLink" # ~~-> IssnLink:FormWidget~~, } diff --git a/portality/forms/application_processors.py b/portality/forms/application_processors.py index a10b585e80..cd7fdaf5f4 100644 --- a/portality/forms/application_processors.py +++ b/portality/forms/application_processors.py @@ -253,7 +253,10 @@ def finalise(self, account, save_target=True, email_alert=True, id=None): # set some administrative data now = dates.now_str() self.target.date_applied = now - self.target.set_application_status(constants.APPLICATION_STATUS_PENDING) + if app.config.get("AUTOCHECK_INCOMING", False): + self.target.set_application_status(constants.APPLICATION_STATUS_POST_SUBMISSION_REVIEW) + else: + self.target.set_application_status(constants.APPLICATION_STATUS_PENDING) self.target.set_owner(account.id) self.target.set_last_manual_update() @@ -280,6 +283,15 @@ def finalise(self, account, save_target=True, email_alert=True, id=None): "application": self.target.data })) + # Kick off the post-submission review + if app.config.get("AUTOCHECK_INCOMING", False): + # FIXME: imports are delayed because of a circular import problem buried in portality.decorators + from portality.tasks.application_autochecks import ApplicationAutochecks + from portality.tasks.helpers import background_helper + background_helper.submit_by_bg_task_type(ApplicationAutochecks, + application=self.target.id, + status_on_complete=constants.APPLICATION_STATUS_PENDING) + class AdminApplication(ApplicationProcessor): """ @@ -704,8 +716,11 @@ def finalise(self, save_target=True, email_alert=True): # if we are allowed to finalise, kick this up to the superclass super(PublisherUpdateRequest, self).finalise() - # set the status to update_request (if not already) - self.target.set_application_status(constants.APPLICATION_STATUS_UPDATE_REQUEST) + # set the status to post submission review (will be updated again later after the review job runs) + if app.config.get("AUTOCHECK_INCOMING", False): + self.target.set_application_status(constants.APPLICATION_STATUS_POST_SUBMISSION_REVIEW) + else: + self.target.set_application_status(constants.APPLICATION_STATUS_UPDATE_REQUEST) # Save the target self.target.set_last_manual_update() @@ -730,6 +745,15 @@ def finalise(self, save_target=True, email_alert=True): else: self.target.remove_current_journal() + # Kick off the post-submission review + if app.config.get("AUTOCHECK_INCOMING", False): + # FIXME: imports are delayed because of a circular import problem buried in portality.decorators + from portality.tasks.application_autochecks import ApplicationAutochecks + from portality.tasks.helpers import background_helper + background_helper.submit_by_bg_task_type(ApplicationAutochecks, + application=self.target.id, + status_on_complete=constants.APPLICATION_STATUS_UPDATE_REQUEST) + # email the publisher to tell them we received their update request if email_alert: try: diff --git a/portality/lib/dataobj.py b/portality/lib/dataobj.py index 47fd7421b4..5f704bff4c 100644 --- a/portality/lib/dataobj.py +++ b/portality/lib/dataobj.py @@ -273,7 +273,7 @@ class DataObj(object): "license": string_canonicalise(["CC BY", "CC BY-NC", "CC BY-NC-ND", "CC BY-NC-SA", "CC BY-ND", "CC BY-SA", "Not CC-like"], allow_fail=True), "persistent_identifier_scheme": string_canonicalise(["None", "DOI", "Handles", "ARK"], allow_fail=True), "format": string_canonicalise(["PDF", "HTML", "ePUB", "XML"], allow_fail=True), - "deposit_policy": string_canonicalise(["None", "Sherpa/Romeo", "Dulcinea", "OAKlist", "Diadorim"], allow_fail=True), + "deposit_policy": string_canonicalise(["None", "Sherpa/Romeo", "Dulcinea", "OAKlist", "Diadorim", "Mir@bel"], allow_fail=True), } def __init__(self, raw=None, struct=None, construct_raw=True, expose_data=False, properties=None, coerce_map=None, construct_silent_prune=False, construct_maintain_reference=False, *args, **kwargs): diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 0bcaf9d916..07b1bb7d6f 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -20,10 +20,7 @@ def remove_fields(query: dict, fields_to_remove: list): def public_query_validator(q): # no deep paging - if q.from_result() > 10000: - return False - - if q.size() > 200: + if (q.from_result()+q.size()) > 10000: return False return True diff --git a/portality/lib/seamless.py b/portality/lib/seamless.py index 2865b7227b..8d44b55ccb 100644 --- a/portality/lib/seamless.py +++ b/portality/lib/seamless.py @@ -495,6 +495,32 @@ def add_to_list(self, path, val, coerce=None, allow_coerce_failure=False, allow_ # otherwise, append current.append(val) + def exists_in_list(self, path, val=None, matchsub=None, apply_struct_on_matchsub=True): + l = self.get_list(path) + + for entry in l: + if val is not None: + if entry == val: + return True + elif matchsub is not None: + # attempt to coerce the sub + if apply_struct_on_matchsub: + try: + type, struct, instructions = self._struct.lookup(path) + if struct is not None: + matchsub = struct.construct(matchsub, struct).data + except: + pass + + matches = 0 + for k, v in matchsub.items(): + if entry.get(k) == v: + matches += 1 + if matches == len(list(matchsub.keys())): + return True + + return False + def delete_from_list(self, path, val=None, matchsub=None, prune=True, apply_struct_on_matchsub=True): """ Note that matchsub will be coerced with the struct if it exists, to ensure diff --git a/portality/migrate/20230712_3519_autochecks/README.md b/portality/migrate/20230712_3519_autochecks/README.md new file mode 100644 index 0000000000..fd4f1d182d --- /dev/null +++ b/portality/migrate/20230712_3519_autochecks/README.md @@ -0,0 +1,6 @@ +Once this feature has been deployed to production, the following steps should be taken to ensure that the autochecks are run on all existing journals and applications: + +```bash +python portality/scripts/autochecks.py -J -A +``` + diff --git a/portality/migrate/20230712_3519_autochecks/__init__.py b/portality/migrate/20230712_3519_autochecks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/migrate/3751_turkey_search/README.md b/portality/migrate/3751_turkey_search/README.md new file mode 100644 index 0000000000..901d66c553 --- /dev/null +++ b/portality/migrate/3751_turkey_search/README.md @@ -0,0 +1,9 @@ +# 26 01 2024; Issue 3751 - Search bug: search results display 'Turkey' even though 'Türkiye' is used in the record + +convert 'Turkey' to 'Türkiye' in Journal's index.country + +## Execution + +Run the migration with + + python portality/upgrade.py -u portality/migrate/3751_turkey_search/migrate.json \ No newline at end of file diff --git a/portality/migrate/3751_turkey_search/__init__.py b/portality/migrate/3751_turkey_search/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/migrate/3751_turkey_search/migrate.json b/portality/migrate/3751_turkey_search/migrate.json new file mode 100644 index 0000000000..df864898fa --- /dev/null +++ b/portality/migrate/3751_turkey_search/migrate.json @@ -0,0 +1,35 @@ +{ + "batch": 10000, + "types": [ + { + "type": "journal", + "init_with_model": true, + "keepalive": "20m", + "functions" : [ + "portality.migrate.3751_turkey_search.operations.correct_turkey" + ], + "query": { + "query": { + "term": { + "index.country.exact": "Turkey" + } + } + } + }, + { + "type": "application", + "init_with_model": true, + "keepalive": "20m", + "functions" : [ + "portality.migrate.3751_turkey_search.operations.correct_turkey" + ], + "query": { + "query": { + "term": { + "index.country.exact": "Turkey" + } + } + } + } + ] +} \ No newline at end of file diff --git a/portality/migrate/3751_turkey_search/operations.py b/portality/migrate/3751_turkey_search/operations.py new file mode 100644 index 0000000000..0c48884d86 --- /dev/null +++ b/portality/migrate/3751_turkey_search/operations.py @@ -0,0 +1,9 @@ +from portality.models.v2.journal import JournalLikeObject + + +def correct_turkey(obj): + obj: JournalLikeObject + if obj.data.get('index', {}).get('country') == 'Turkey': + print(f'fix {obj.id}, {obj.bibjson().title} ') + obj.data['index']['country'] = 'Türkiye' + return obj diff --git a/portality/models/__init__.py b/portality/models/__init__.py index 57eb28e5f9..eea1859a70 100644 --- a/portality/models/__init__.py +++ b/portality/models/__init__.py @@ -2,7 +2,7 @@ # import the versioned objects, so that the current version is the default one from portality.models.v2 import shared_structs from portality.models.v2.bibjson import JournalLikeBibJSON -from portality.models.v2.journal import Journal, JournalQuery, IssnQuery, PublisherQuery, TitleQuery, ContinuationException +from portality.models.v2.journal import JournalLikeObject, Journal, JournalQuery, IssnQuery, PublisherQuery, TitleQuery, ContinuationException from portality.models.v2.application import Application, SuggestionQuery, OwnerStatusQuery, DraftApplication, AllPublisherApplications from portality.models.v2.application import Application as Suggestion @@ -27,6 +27,7 @@ from portality.models.harvester import HarvestState from portality.models.event import Event from portality.models.notifications import Notification +from portality.models.autocheck import Autocheck import sys diff --git a/portality/models/autocheck.py b/portality/models/autocheck.py new file mode 100644 index 0000000000..c653ba228d --- /dev/null +++ b/portality/models/autocheck.py @@ -0,0 +1,215 @@ +from portality.dao import DomainObject +from portality.lib.seamless import SeamlessMixin +from portality.lib.coerce import COERCE_MAP +from portality.lib import es_data_mapping +from portality.core import app + +import json +from copy import deepcopy + + +AUTOCHECK_STRUCT = { + "fields": { + "id": {"coerce": "unicode"}, + "es_type": {"coerce": "unicode"}, + "created_date": {"coerce": "utcdatetime"}, + "last_updated": {"coerce": "utcdatetime"}, + "application": {"coerce": "unicode"}, + "journal": {"coerce": "unicode"} + }, + "lists": { + "checks": {"contains": "object"} + }, + + "structs": { + "checks": { + "fields": { + "id": {"coerce": "unicode"}, + "field": {"coerce": "unicode"}, + "original_value": {"coerce": "unicode"}, + "replaced_value": {"coerce": "unicode"}, + "advice": {"coerce": "unicode"}, + "reference_url": {"coerce": "unicode"}, + "checked_by": {"coerce": "unicode"}, + "dismissed": {"coerce": "bool"}, + "context": {"coerce": "unicode"} + }, + "lists": { + "suggested_value": {"contains": "field", "coerce": "unicode"} + } + } + } +} + +MAPPING_OPTS = { + "dynamic": None, + "coerces": app.config["DATAOBJ_TO_MAPPING_DEFAULTS"] +} + + +class Autocheck(SeamlessMixin, DomainObject): + __type__ = "autocheck" + + __SEAMLESS_STRUCT__ = [ + AUTOCHECK_STRUCT + ] + + __SEAMLESS_COERCE__ = COERCE_MAP + + def __init__(self, **kwargs): + # FIXME: hack, to deal with ES integration layer being improperly abstracted + if "_source" in kwargs: + kwargs = kwargs["_source"] + super(Autocheck, self).__init__(raw=kwargs) + + def mappings(self): + return es_data_mapping.create_mapping(self.__seamless_struct__.raw, MAPPING_OPTS) + + @property + def data(self): + return self.__seamless__.data + + @classmethod + def for_application(cls, app_id): + q = ApplicationQuery(app_id) + res = cls.object_query(q.query()) + if len(res) > 0: + return res[0] + return None + + @classmethod + def for_journal(cls, journal_id): + q = JournalQuery(journal_id) + res = cls.object_query(q.query()) + if len(res) > 0: + return res[0] + return None + + @classmethod + def delete_all_but_latest(cls, journal_id=None, application_id=None): + if journal_id is not None: + q = JournalQuery(journal_id) + elif application_id is not None: + q = ApplicationQuery(application_id) + else: + return + + res = cls.object_query(q.query()) + if len(res) > 1: + for r in res[1:]: + r.delete() + + @property + def application(self): + return self.__seamless__.get_single("application") + + @application.setter + def application(self, val): + self.__seamless__.set_with_struct("application", val) + + @property + def journal(self): + return self.__seamless__.get_single("journal") + + @journal.setter + def journal(self, val): + self.__seamless__.set_with_struct("journal", val) + + def add_check(self, field=None, original_value=None, suggested_value=None, advice=None, reference_url=None, context=None, checked_by=None): + obj = {} + if field is not None: + obj["field"] = field + if original_value is not None: + obj["original_value"] = original_value + if suggested_value is not None: + if not isinstance(suggested_value, list): + suggested_value = [suggested_value] + obj["suggested_value"] = suggested_value + if advice is not None: + obj["advice"] = advice + if reference_url is not None: + obj["reference_url"] = reference_url + if checked_by is not None: + obj["checked_by"] = checked_by + if context is not None: + obj["context"] = json.dumps(context) + + # ensure we add the check only once + exists = self.__seamless__.exists_in_list("checks", matchsub=obj) + + # now give this check an id and add it + if not exists: + obj["id"] = self.makeid() + self.__seamless__.add_to_list_with_struct("checks", obj) + + # return the constructed object, in case the caller could use it + return obj + + @property + def checks(self): + annos = self.__seamless__.get_list("checks") + realised_checks = [] + for anno in annos: + anno = deepcopy(anno) + if "context" in anno: + anno["context"] = json.loads(anno["context"]) + realised_checks.append(anno) + return realised_checks + + @property + def checks_raw(self): + return self.__seamless__.get_list("checks") + + def dismiss(self, check_id): + annos = self.checks_raw + for anno in annos: + if anno.get("id") == check_id: + anno["dismissed"] = True + break + + def undismiss(self, check_id): + annos = self.checks_raw + for anno in annos: + if anno.get("id") == check_id: + del anno["dismissed"] + break + + +class ApplicationQuery(object): + def __init__(self, app_id): + self._app_id = app_id + + def query(self): + return { + "query" : { + "bool": { + "must": [ + {"term": {"application.exact": self._app_id}} + ] + } + }, + "size": 1, + "sort": { + "created_date": {"order": "desc"} + } + } + + +class JournalQuery(object): + def __init__(self, journal_id): + self._journal_id = journal_id + + def query(self): + return { + "query" : { + "bool": { + "must": [ + {"term": {"journal.exact": self._journal_id}} + ] + } + }, + "size": 1, + "sort": { + "created_date": {"order": "desc"} + } + } \ No newline at end of file diff --git a/portality/models/openurl.py b/portality/models/openurl.py index b47962e37b..f8644a7005 100644 --- a/portality/models/openurl.py +++ b/portality/models/openurl.py @@ -126,22 +126,8 @@ def get_result_url(self): # (assuming that the user gave us specific enough information ident = journal.id - # If there request has a volume parameter, query for presence of an article with that volume - if self.volume: - vol_iss_results = self.query_for_vol(journal) - - if vol_iss_results == None: - # we were asked for a vol/issue, but weren't given the correct information to get it. - return None - elif vol_iss_results['hits']['total']['value'] > 0: - # construct the toc url using the ident, plus volume and issue - jtoc_url = url_for("doaj.toc", identifier=ident, volume=self.volume, issue=self.issue) - else: - # If no results, the DOAJ does not contain the vol/issue being searched. (Show openurl 404) - jtoc_url = None - else: - # if no volume parameter, construct the toc url using the ident only - jtoc_url = url_for("doaj.toc", identifier=ident) + # construct the toc url using the ident only + jtoc_url = url_for("doaj.toc", identifier=ident) return jtoc_url #~~->Article:Page~~ diff --git a/portality/regex.py b/portality/regex.py index a298a4731f..09a55a52bf 100644 --- a/portality/regex.py +++ b/portality/regex.py @@ -21,6 +21,7 @@ r'^(?:https?)://' # Scheme: http(s) or ftp r'(?:[\w-]+\.)*[\w-]+' # Domain name (optional subdomains) r'(?:\.[a-z]{2,})' # Top-level domain (e.g., .com, .org) + r'(?:\:(0|6[0-5][0-5][0-3][0-5]|[1-5][0-9][0-9][0-9][0-9]|[1-9][0-9]{0,3}))?' # port (0-65535) preceded with `:` r'(?:\/[^\/\s]*)*' # Path (optional) r'(?:\?[^\/\s]*)?' # Query string (optional) r'(?:#[^\/\s]*)?$' # Fragment (optional) diff --git a/portality/scripts/3821_add_deposit_policy/Mirabel-deposit-policy.csv b/portality/scripts/3821_add_deposit_policy/Mirabel-deposit-policy.csv new file mode 100644 index 0000000000..8c6c96bed7 --- /dev/null +++ b/portality/scripts/3821_add_deposit_policy/Mirabel-deposit-policy.csv @@ -0,0 +1,170 @@ +Journal title,Journal ISSN (print version),Journal EISSN (online version),Deposit policy directory,URL for deposit policy +20 & 21. Revue d'histoire,2649-664X,2649-6100,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9431 +Agora débats/jeunesses,1268-5666,1968-3758,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1683 +Champs de Mars,1253-1871,2427-3244,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1283 +Critique internationale,1290-7839,1777-554X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/45 +Raisons politiques : études de pensée politique,1291-1941,1950-6708,Mir@bel,https://reseau-mirabel.info/revue/titre-id/580 +Revue économique,0035-2764,1950-6694,Mir@bel,https://reseau-mirabel.info/revue/titre-id/85 +Revue française de science politique,0035-2950,1950-6686,Mir@bel,https://reseau-mirabel.info/revue/titre-id/25 +"Vingtième siècle, revue d'histoire",0294-1759,1950-6678,Mir@bel,https://reseau-mirabel.info/revue/titre-id/98 +Année du Maghreb,1952-8108,2109-9405,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1725 +Anglophonia,,2427-0466,Mir@bel,https://reseau-mirabel.info/revue/titre-id/16924 +Caliban : French journal of English studies,2425-6250,2431-1766,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3793 +Caravelle,1147-6753,2272-9828,Mir@bel,https://reseau-mirabel.info/revue/titre-id/854 +Cinémas d'Amérique latine,1267-4397,2425-1356,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1551 +Criticón,0247-381X,2272-9852,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2591 +"Diasporas : circulations, migrations, histoire",1637-5823,2431-1472,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3832 +Dossiers des sciences de l'éducation : revue internationale des sciences de l'éducation,1296-2104,2272-9968,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2331 +"Histoire, médecine et santé",2263-8911,2557-2113,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4463 +Littératures,0563-9751,2273-0311,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2594 +P@lethnologie,,2108-6532,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5173 +Pallas,0031-0387,2272-7639,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2597 +Sciences de la société,1168-1446,2275-2145,Mir@bel,https://reseau-mirabel.info/revue/titre-id/362 +Sud-Ouest européen,1276-4930,2273-0257,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2579 +Population,0032-4663,1957-7966,Mir@bel,https://reseau-mirabel.info/revue/titre-id/573 +Population (english edition),1634-2941,1958-9190,Mir@bel,https://reseau-mirabel.info/revue/titre-id/572 +Biens symboliques / Symbolic Goods,,2490-9424,Mir@bel,https://reseau-mirabel.info/revue/titre-id/7535 +"Extrême-Orient, Extrême-Occident",0754-5010,2108-7105,Mir@bel,https://reseau-mirabel.info/revue/titre-id/846 +Hybrid : Revue des arts et médiations humaines = Journal of arts and human mediations,,2276-3538,Mir@bel,https://reseau-mirabel.info/revue/titre-id/7513 +Marges : revue d'art contemporain,1767-7114,2416-8742,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2902 +Médiévales,0751-2708,1777-5892,Mir@bel,https://reseau-mirabel.info/revue/titre-id/682 +Recherches linguistiques de Vincennes,0986-6124,1958-9239,Mir@bel,https://reseau-mirabel.info/revue/titre-id/691 +Photographica,2780-8572,2740-5826,Mir@bel,https://reseau-mirabel.info/revue/titre-id/11313 +Revue d'histoire des sciences humaines,1622-468X,1963-1022,Mir@bel,https://reseau-mirabel.info/revue/titre-id/597 +Revue internationale des études du développement,2554-3415,2554-3555,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4954 +Socio-anthropologie,1276-8707,1773-018X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/703 +Revue management & avenir,1768-5958,1969-6574,Mir@bel,https://reseau-mirabel.info/revue/titre-id/784 +Moussons : Recherches en sciences humaines sur l'Asie du Sud-Est,1620-3224,2262-8363,Mir@bel,https://reseau-mirabel.info/revue/titre-id/978 +Bulletin archéologique des Écoles françaises à l'étranger,,2732-687X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/11406 +Bulletin de Correspondance Hellénique,0007-4217,2241-0104,Mir@bel,https://reseau-mirabel.info/revue/titre-id/728 +Bulletin de correspondance hellénique moderne et contemporain,,2732-6535,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9294 +Cahiers du Centre d'Études Chypriotes,0761-8271,2647-7300,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4579 +Flux,1154-2721,1958-9557,Mir@bel,https://reseau-mirabel.info/revue/titre-id/543 +Revue Européenne des Migrations Internationales,0765-0752,1777-5418,Mir@bel,https://reseau-mirabel.info/revue/titre-id/697 +Amnis,,1764-7193,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1090 +Dialogues d'histoire ancienne,0755-7256,1955-270X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/733 +Dialogues d'histoire ancienne. Supplément,2108-1433,2554-4098,Mir@bel,https://reseau-mirabel.info/revue/titre-id/7640 +"Football(s). Histoire, culture, économie, société",2967-0837,2968-0115,Mir@bel,https://reseau-mirabel.info/revue/titre-id/18921 +Philosophique,0980-0891,2259-4574,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1744 +Philosophique. Hors-série,2968-692X,,Mir@bel,https://reseau-mirabel.info/revue/titre-id/20104 +Publications mathématiques de Besançon - Algèbre et Théorie des nombres,2804-8504,2592-6616,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6266 +Semen. Revue de sémio-linguistique des textes et discours,0761-2990,1957-780X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/701 +Skén&graphie,2272-0642,2553-1875,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4699 +European Bulletin of Himalayan Research,0943-8254,2823-6114,Mir@bel,https://reseau-mirabel.info/revue/titre-id/842 +Revue LISA / LISA e-journal,,1762-6153,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1156 +M@n@gement,,1286-4692,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1395 +Cahiers de la recherche sur les droits fondamentaux,1634-8842,2264-1246,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4582 +Cahiers de philosophie de l'Université de Caen,1282-6545,2677-6529,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5394 +"Discours : Revue de linguistique, psycholinguistique et informatique",,1963-1723,Mir@bel,https://reseau-mirabel.info/revue/titre-id/659 +Double jeu,1762-0597,2610-072X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4906 +Études irlandaises,0183-973X,2259-8863,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1723 +Kentron,0765-0590,2264-1459,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5396 +Questions de style,,1769-6909,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5388 +Syntaxe et sémantique,1623-6742,2271-2852,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2101 +Tabularia : sources écrites des mondes normands médiévaux,1951-2562,1630-7364,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4720 +"Télémaque : philosophie, éducation, société",1263-588X,2118-2191,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1446 +Transalpina,1278-334X,2534-5184,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9324 +Journal of Interdisciplinary Methodologies and Issues in Science,,2430-3038,Mir@bel,https://reseau-mirabel.info/revue/titre-id/14756 +Bretagne Linguistique,1270-2412,2727-9383,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10510 +Culture & Musées,1766-2923,2111-4528,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1069 +Annales mathématiques Blaise Pascal,1259-1734,2118-7436,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5759 +K@iros : revue transdisciplinaire en sciences de l'information et de la communication et civilisations étrangères,,2492-1599,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3941 +Revue du Centre Michel de l'Hospital,,2273-872X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9522 +Siècles,1266-6726,2275-2129,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1719 +Sociopoétiques,,2497-3610,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8820 +Viatica,,2275-0827,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3937 +Anthropologie & Santé,,2111-5028,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1731 +Études créoles,0708-2398,2607-8988,Mir@bel,https://reseau-mirabel.info/revue/titre-id/12352 +Travaux interdisciplinaires sur la parole et le langage,,2264-7082,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1918 +"Ambiances : Environnement sensible, architecture et espace urbain",,2266-839X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2053 +Bulletin du Centre d'études médiévales d'Auxerre,,1954-3093,Mir@bel,https://reseau-mirabel.info/revue/titre-id/638 +Bulletin du Centre d'études médiévales d'Auxerre. Hors-série,,1961-8875,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4419 +Crescentis : Revue internationale d'Histoire de la vigne et du vin,,2647-4840,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6953 +Éclats. Revue des doctorantes et doctorants de l'école doctorale 592 LECLA,,2804-5866,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15219 +Interfaces : Image - Texte - Language,1164-6225,2647-6754,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6952 +Sciences humaines combinées : revue électronique des écoles doctorales ED LISIT et ED LETS,,1961-9936,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4417 +Territoires du vin,,1760-5296,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4420 +Textes & contextes,,1961-991X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2244 +Bulletin des arrêts de la cour d'appel de Lyon,,2607-866X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/7507 +Cahiers Jean Moulin,,2553-9221,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6135 +Carnets du LARHRA,2260-7633,2648-1782,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6496 +ELAD-SILDA : Études de Linguistique et d'Analyse des Discours – Studies in Linguistics and Discourse Analysis,,2609-6609,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6896 +Lexis : Journal in English Lexicology,,1951-6215,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4545 +Modernités russes,0292-0328,2725-2124,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10667 +Nouveaux cahiers de Marge,,2607-4427,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6893 +Pratiques & formes littéraires 16-18,,2534-7683,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10665 +Revue internationale des francophonies,,2556-1944,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6215 +"Itinéraires. Littérature, textes, cultures",2100-1340,2427-920X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3073 +Recherches en éducation,,1954-3077,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3150 +Strenae. Recherches sur les livres et les objets culturels de l'enfance,,2109-9081,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2035 +Cahiers scientifiques du transport,1150-8809,2804-2107,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4526 +Orientation scolaire et professionnelle,0249-6739,2104-3795,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1142 +Savoirs en lien,,2968-0263,Mir@bel,https://reseau-mirabel.info/revue/titre-id/18946 +Recherches anglaises et nord-américaines,0557-6989,3000-4411,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6237 +Revue des sciences sociales,1623-6572,2107-0385,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4072 +Chrétiens et sociétés XVIe-XXIe siècles,2267-7143,1965-0809,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2664 +Annales de l'Institut Fourier,0373-0956,1777-5310,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5386 +"Glad! : revue sur le langage, le genre, les sexualités",,2551-0819,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6154 +MathematicS in action,,2102-5754,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6206 +SMAI-JCM. The SMAI Journal of Computational Mathematics,,2426-8399,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8348 +Sociologie du travail,0038-0296,1777-5701,Mir@bel,https://reseau-mirabel.info/revue/titre-id/316 +Sources. Matériaux & terrains en études africaines,,2708-7034,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10421 +International Review of Public Policy,2679-3873,2706-6274,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8326 +Mathematics Research Reports,,2772-9559,Mir@bel,https://reseau-mirabel.info/revue/titre-id/12095 +Annales Henri Lebesgue,,2644-9463,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8347 +Humanités numériques,,2736-2337,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8397 +Esclavages & Post-esclavages,,2540-6647,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9210 +Déméter : théories et pratiques artistiques contemporaines,,1638-556X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10192 +grandes figures historiques dans les lettres et les arts,,2261-0871,Mir@bel,https://reseau-mirabel.info/revue/titre-id/7523 +Lexique,0756-7138,2804-7397,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10207 +Mosaïque,,2105-1100,Mir@bel,https://reseau-mirabel.info/revue/titre-id/20562 +Sciences Eaux & Territoires,2109-3016,1775-3783,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2048 +Revue du Rhin supérieur,2681-6792,2803-9513,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10673 +"Frontière·s. Revue d'archéologie, histoire & histoire de l'art",,2534-7535,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9469 +Comptes rendus. Biologies,1631-0691,1768-3238,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5465 +Comptes rendus. Chimie,1631-0748,1878-1543,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5464 +Comptes rendus. Géoscience,1631-0713,1778-7025,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5466 +Comptes rendus. Mathématique,1631-073X,1778-3569,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5467 +Comptes rendus. Mécanique,1631-0721,1873-7234,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5468 +Comptes rendus. Physique,1631-0705,1878-1535,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5672 +Archipélies,2110-7130,2777-5909,Mir@bel,https://reseau-mirabel.info/revue/titre-id/11812 +Contextes et Didactiques,,2551-6116,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5676 +Études caribéennes,1779-0980,1961-859X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/665 +Voix contemporaines,,2801-2321,Mir@bel,https://reseau-mirabel.info/revue/titre-id/11135 +Confluentes Mathematici,1793-7442,1793-7434,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5709 +Actualité juridique du dommage corporel,,2497-2118,Mir@bel,https://reseau-mirabel.info/revue/titre-id/9762 +Cahiers Fablijes,,2999-9154,Mir@bel,https://reseau-mirabel.info/revue/titre-id/22021 +Canal psy,1253-9392,2777-2055,Mir@bel,https://reseau-mirabel.info/revue/titre-id/14732 +Parcours anthropologiques,1634-7706,2273-0362,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2592 +Textures,1253-5044,2971-4109,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15304 +Revue francophone sur la santé et les territoires,,2492-3672,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4891 +Populations vulnérables,2269-0182,2650-7684,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6155 +Territoires contemporains,,1961-9944,Mir@bel,https://reseau-mirabel.info/revue/titre-id/2459 +Transversales,,2273-1806,Mir@bel,https://reseau-mirabel.info/revue/titre-id/4764 +Peer Community Journal,,2804-3871,Mir@bel,https://reseau-mirabel.info/revue/titre-id/16359 +TheoretiCS,,2751-4838,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15330 +Management & Organisations du Sport,,2804-8598,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15332 +Mémoires de l'Institut de préhistoire et d'archéologie Alpes Méditerranée,1286-4374,,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15650 +Carnets Botaniques,,2727-6287,Mir@bel,https://reseau-mirabel.info/revue/titre-id/15933 +"Afriques. Débats, méthodes et terrains d'histoire",,2108-6796,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1086 +Algebraic combinatorics,,2589-5486,Mir@bel,https://reseau-mirabel.info/revue/titre-id/8346 +Annales de la Faculté des sciences de Toulouse,0240-2963,2258-7519,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5493 +Journal de théorie des nombres de Bordeaux,1246-7405,2118-8572,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6047 +Open Geomechanics,,2644-9676,Mir@bel,https://reseau-mirabel.info/revue/titre-id/10240 +Open Journal of Mathematical Optimization,,2777-5860,Mir@bel,https://reseau-mirabel.info/revue/titre-id/12091 +Revue Ouverte d'Intelligence Artificielle,,2967-9672,Mir@bel,https://reseau-mirabel.info/revue/titre-id/12087 +Journal de l'École polytechnique - Mathématiques,2429-7100,2270-518X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/6030 +Cahiers de l'AREFLE,2728-5332,2119-615X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/16859 +Didactique du FLES,2966-957X,2826-777X,Mir@bel,https://reseau-mirabel.info/revue/titre-id/17694 +Source(s),2265-1306,2261-8562,Mir@bel,https://reseau-mirabel.info/revue/titre-id/16864 +RadaR,,2825-9696,Mir@bel,https://reseau-mirabel.info/revue/titre-id/16927 +ABE Journal : European architecture beyond Europe,,2275-6639,Mir@bel,https://reseau-mirabel.info/revue/titre-id/3695 +Cahiers du genre,1298-6046,1968-3928,Mir@bel,https://reseau-mirabel.info/revue/titre-id/1253 +Catalonia,,1760-6659,Mir@bel,https://reseau-mirabel.info/revue/titre-id/17600 +Iberic@l,,2260-2534,Mir@bel,https://reseau-mirabel.info/revue/titre-id/17682 +Amplitude du droit,,2826-1305,Mir@bel,https://reseau-mirabel.info/revue/titre-id/17685 +Pensée d'Ailleurs,,2826-9497,Mir@bel,https://reseau-mirabel.info/revue/titre-id/17720 +À tradire. Didactique de la traduction pragmatique et de la communication technique,,2968-3912,Mir@bel,https://reseau-mirabel.info/revue/titre-id/19603 +Carnets de Poédiles,,2970-3174,Mir@bel,https://reseau-mirabel.info/revue/titre-id/19924 +Cahiers François Viète,1297-9112,2780-9986,Mir@bel,https://reseau-mirabel.info/revue/titre-id/5135 +Projectics / Proyéctica / Projectique,2031-9703,2032-0043,Mir@bel,https://reseau-mirabel.info/revue/titre-id/787 diff --git a/portality/scripts/3821_add_deposit_policy/add_deposit_policy.py b/portality/scripts/3821_add_deposit_policy/add_deposit_policy.py new file mode 100644 index 0000000000..203a1270ff --- /dev/null +++ b/portality/scripts/3821_add_deposit_policy/add_deposit_policy.py @@ -0,0 +1,59 @@ +""" +The CSV contains a list of journals; some of them are indexed DOAJ. Updates the author repository policy answers for those that are indexed. + +* identify the journals from the list that are indexed in DOAJ +* update the repository policy questions with the new answer: Mir@bel, plus URL +* don't change the manual last updated date +* output a list of journals updated successfully + +""" + +import csv +from copy import deepcopy + +from portality.models import Journal + + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser() + + parser.add_argument("--save", help="Apply the changes to the Journal record", action='store_true') + args = parser.parse_args() + + with open('Mirabel-deposit-policy.csv', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + + with open('add-Mirabel-deposit-policy.csv', 'w') as g: + writer = csv.DictWriter(g, fieldnames=list(reader.fieldnames) + ['found', 'existing deposit policy', + 'existing deposit policy url', 'added']) + writer.writeheader() + + for r in reader: + pissn = r.get('Journal ISSN (print version)', None) + eissn = r.get('Journal EISSN (online version)', None) + + s = deepcopy(r) + + try: + j = Journal.find_by_issn_exact([pissn, eissn])[0] + s['found'] = j.id + s['existing deposit policy'] = j.bibjson().deposit_policy + s['existing deposit policy url'] = j.bibjson().deposit_policy_url + + if args.save: + # Do deposit update + dp = r.get('Deposit policy directory') + if dp not in j.bibjson().deposit_policy: + j.bibjson().add_deposit_policy(r.get('Deposit policy directory')) + j.bibjson().deposit_policy_url = r.get('URL for deposit policy') + + j.prep() + j.save(blocking=True) + s['added'] = 'y' + + except IndexError: + pass # Nothing to do, it's not in the DOAJ index + + writer.writerow(s) diff --git a/portality/scripts/anon_import.py b/portality/scripts/anon_import.py index c821444ad6..5740a9fb29 100644 --- a/portality/scripts/anon_import.py +++ b/portality/scripts/anon_import.py @@ -4,16 +4,20 @@ Configure the target index in your *.cfg override file For now, this import script requires the same index pattern (prefix, 'types', index-per-type setting) as the exporter. +Will ignore your setting STORE_IMPL in app.cfg - defaults to s3, alternatively use local storage via [-s local] + E.g. for dev: +DOAJENV=dev python portality/scripts/anon_import.py data_import_settings/dev_basics.json -Ensure in dev.cfg you've set STORE_IMPL = "portality.store.StoreS3" -python portality/scripts/anon_import.py data_import_settings/dev_basics.json +or for a test server: +DOAJENV=test python portality/scripts/anon_import.py data_import_settings/test_server.json """ import esprit, json, gzip, shutil, elasticsearch from portality.core import app, es_connection, initialise_index from portality.store import StoreFactory from portality.util import ipt_prefix +from doajtest.helpers import patch_config # FIXME: monkey patch for esprit.bulk (but esprit's chunking is handy) @@ -95,7 +99,8 @@ def do_import(config): print(("Importing from {x}".format(x=filename))) imported_count = esprit.tasks.bulk_load(es_connection, ipt_prefix(import_type), uncompressed_file, - limit=limit, max_content_length=config.get("max_content_length", 100000000)) + limit=limit, + max_content_length=config.get("max_content_length", 100000000)) tempStore.delete_file(container, filename) if limit is not None and imported_count != -1: @@ -114,15 +119,32 @@ def do_import(config): import argparse parser = argparse.ArgumentParser() - parser.add_argument("config", help="Config file for import run") + parser.add_argument("config", help="Config file for import run, e.g dev_basics.json") + parser.add_argument('-s', '--storeimpl', + help="Use S3 (default) or StoreLocal as anon data source", + choices=['s3', 'local'], + default='s3', + required=False) args = parser.parse_args() with open(args.config, "r", encoding="utf-8") as f: - config = json.loads(f.read()) + cf = json.loads(f.read()) # FIXME: monkey patch for esprit raw_bulk unwanted_primate = esprit.raw.raw_bulk esprit.raw.raw_bulk = es_bulk - do_import(config) + if args.storeimpl == 'local': + print("\n**\nImporting from Local storage") + original_configs = patch_config(app, { + 'STORE_IMPL': "portality.store.StoreLocal" + }) + else: + print("\n**\nImporting from S3 storage") + original_configs = patch_config(app, { + 'STORE_IMPL': "portality.store.StoreS3" + }) + + do_import(cf) esprit.raw.raw_bulk = unwanted_primate + patch_config(app, original_configs) diff --git a/portality/scripts/autochecks.py b/portality/scripts/autochecks.py new file mode 100644 index 0000000000..a142a833e5 --- /dev/null +++ b/portality/scripts/autochecks.py @@ -0,0 +1,102 @@ +from portality.bll import DOAJ +from portality import models +from portality.lib import dates +import csv + + +if __name__ == '__main__': + + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument("-i", "--id", help="ID of application or journal on which to run autocheck") + parser.add_argument("-a", "--applications", help="CSV of application IDs on which to run autochecks") + parser.add_argument("-j", "--journals", help="CSV of journal IDs on which to run autochecks") + parser.add_argument("-A", "--all_applications", help="Run autochecks on all applications", action="store_true") + parser.add_argument("-J", "--all_journals", help="Run autochecks on all journals", action="store_true") + + args = parser.parse_args() + if args.id is None and args.applications is None and args.journals is None and args.all_applications is None and args.all_journals is None: + print("You must specify an application id with the -i argument, or a list of ids in a csv with the -a/A or -j/J arguments for applications and journals respectively") + exit(1) + + if args.id is not None and (args.applications is not None or args.all_applications is not None or args.journals is not None or args.all_journals is not None): + print("You cannot specify both -i and -a/A or -j/J") + exit(1) + + if args.applications is not None and args.all_applications is not None: + print("You cannot specify both -a and -A") + exit(1) + + if args.journals is not None and args.all_journals is not None: + print("You cannot specify both -j and -J") + exit(1) + + anno_svc = DOAJ.autochecksService() + + logger = lambda x: print(dates.now_str_with_microseconds() + " " + x) + + if args.id: + application = models.Application.pull(args.id) + if application is not None: + print("\nAnnotating application {x}".format(x=application.id)) + anno_svc.autocheck_application(application, logger=logger) + exit(1) + + journal = models.Journal.pull(args.id) + if journal is not None: + print("\nAnnotating journal {x}".format(x=journal.id)) + anno_svc.autocheck_journal(journal, logger=logger) + exit(1) + + print("ID did not resolve to an application or journal") + exit(1) + + if args.all_applications: + print("Autochecking all applications") + anno_svc.autocheck_applications(logger=logger) + + if args.all_journals: + print("Autochecking all journals") + anno_svc.autocheck_journals(logger=logger) + + if args.applications: + print("Autochecking applications from {x}".format(x=args.applications)) + with open(args.applications, "r", encoding="utf-8") as f: + reader = csv.reader(f) + for row in reader: + application = None + id = row[0].strip() + if len(id) == 9: + applications = models.Application.find_by_issn(id) + if len(applications) > 0: + application = applications[0] + else: + application = models.Application.pull(id) + + if application is None: + print("Application ID {x} did not resolve to an application".format(x=id)) + continue + print("\nAnnotating application {x}".format(x=application.id)) + anno_svc.autocheck_application(application, logger=lambda x: print(x)) + + if args.journals: + print("Autochecking journals from {x}".format(x=args.journals)) + with open(args.journals, "r", encoding="utf-8") as f: + reader = csv.reader(f) + for row in reader: + journal = None + id = row[0].strip() + if len(id) == 9: + journals = models.Journal.find_by_issn(id) + if len(journals) > 0: + journal = journals[0] + else: + journal = models.Journal.pull(id) + + if journal is None: + print("Journal ID {x} did not resolve to a journal".format(x=id)) + continue + print("\nAnnotating journal {x}".format(x=journal.id)) + anno_svc.autocheck_journal(journal, logger=lambda x: print(x)) \ No newline at end of file diff --git a/portality/scripts/journals_update_via_csv.py b/portality/scripts/journals_update_via_csv.py index c87f252612..f09ae5ea86 100644 --- a/portality/scripts/journals_update_via_csv.py +++ b/portality/scripts/journals_update_via_csv.py @@ -9,15 +9,20 @@ Input CSV is a standard DOAJ journal CSV -Usage: e.g. for a dry-run first with a malformed CSV -export DOAJ_CSV_NOTE='UR autogenerated after publisher CSV uploaded. Change to URL and/or APCS.' -DOAJENV=production python -u journals_update_via_csv.py -i -a adminuser -d | tee +Usage: +e.g. for a dry-run first with a malformed CSV + export DOAJ_CSV_NOTE='UR autogenerated after publisher CSV uploaded. Change to URL and/or APCS.' + DOAJENV=production python -u journals_update_via_csv.py -i -s -d | tee Check the report for errors and the output for expected changes, then run without -d to apply the updates. + +e.g. to validate a CSV upload and create a batch of update requests from that user + DOAJENV=production python -u journals_update_via_csv.py -i -a -m """ import os import csv +import re from portality import lock, constants from portality.core import app @@ -45,10 +50,16 @@ parser = argparse.ArgumentParser() parser.add_argument("-i", "--infile", help="Path to journal update spreadsheet", required=True) - parser.add_argument("-a", "--account", help="account ID of the user to import as", required=True) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-a", "--account", help="Account ID of the user to import as") + group.add_argument('-s', '--sys', help="Validate and create URs using the system user (for admin / batch uploads)", action='store_true') + group.add_argument('-p', '--prefix', help="Read the account by splitting the filename (supplied as e.g. 12341234.publisher_doajupload.csv)", action='store_true') + parser.add_argument("-d", "--dry-run", help="Run this script without actually making index changes", action='store_true') parser.add_argument("-m", "--manual-review", help="Don't finalise the update requests, instead leave open for manual review.", action='store_true') parser.add_argument("-y", "--yes", help="Bypass prompt to accept the note", action='store_true') + parser.add_argument("-f", "--force", help="Ignore warnings and process changes anyway (errors still halt)", action='store_true') args = parser.parse_args() @@ -69,17 +80,33 @@ # Turn off debug so we don't get extra messages in the output app.config['DEBUG'] = False - acc = Account.pull(args.account) - if acc is None: - print(f'ERROR: Account {args.account} not found.') - exit(1) + if args.sys: + acc = sys_acc + else: + found_account_name = re.split(r'[._]', os.path.basename(args.infile))[0] if args.prefix else args.account + print(f'Creating Update Requests as account {found_account_name}') + + acc = Account.pull(found_account_name) + + if acc is None: + print(f'ERROR: Account {args.account} not found.') + exit(1) appSvc = DOAJ.applicationService() validation_results = appSvc.validate_update_csv(args.infile, acc) if validation_results.has_errors_or_warnings(): - print(f'ERROR: CSV validation failed.') + print(f'ERROR: CSV validation failed with warnings or errors.') print(validation_results.json(indent=2)) - exit(1) + + if not args.dry_run: + if not validation_results.has_errors() and args.force: + print('Forcing update despite warnings...') + elif validation_results.has_errors() and args.force: + print("Can't force update on file with errors.") + exit(1) + else: + print(f'No updates processed due to errors or warnings. Supply -f arg to ignore warnings.') + exit(1) # if we get to here, the records can all be imported, so we can go ahead with minimal # additional checks @@ -121,7 +148,8 @@ if len(updates) == 0: print("No updates to do") continue - + + [print(upd) for upd in updates] # Create an update request for this journal update_req = None @@ -164,7 +192,7 @@ # This is the update request, in 'update request' state update_req_for_review = fc.target - # Create an Admin update request review form - portality.forms.application_processors.AdminApplication + # Admin update request review form - portality.forms.application_processors.AdminApplication # ~~ ^->ManEdApplication:FormContext ~~ formulaic_context2 = ApplicationFormFactory.context("admin") fc2 = formulaic_context2.processor( diff --git a/portality/settings.py b/portality/settings.py index 37729773c2..744e4b1108 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -9,7 +9,7 @@ # Application Version information # ~~->API:Feature~~ -DOAJ_VERSION = "6.6.2" +DOAJ_VERSION = "6.6.10" API_VERSION = "3.0.1" ###################################### @@ -73,7 +73,7 @@ ES_TERMS_LIMIT = 1024 -ES_READ_TIMEOUT = '1m' +ES_READ_TIMEOUT = '2m' ##################################################### # Elastic APM config (MUST be configured in env file) @@ -390,6 +390,7 @@ ASSOC_ED_IDLE_WEEKS = 3 # Which statuses the notification queries should be filtered to show +# ~~-> ApplicationStatuses:Config~~ MAN_ED_NOTIFICATION_STATUSES = [ constants.APPLICATION_STATUS_PENDING, constants.APPLICATION_STATUS_IN_PROGRESS, constants.APPLICATION_STATUS_COMPLETED, @@ -451,7 +452,9 @@ HUEY_TASKS = { "ingest_articles": {"retries": 10, "retry_delay": 15}, - "preserve": {"retries": 0, "retry_delay": 15} + "preserve": {"retries": 0, "retry_delay": 15}, + "application_autochecks": {"retries": 0, "retry_delay": 15}, + "journal_autochecks": {"retries": 0, "retry_delay": 15} } #################################### @@ -482,7 +485,8 @@ "portality.models.Application", # ~~->Application:Model~~ "portality.models.DraftApplication", # ~~-> DraftApplication:Model~~ "portality.models.harvester.HarvestState", # ~~->HarvestState:Model~~ - "portality.models.background.BackgroundJob" # ~~-> BackgroundJob:Model~~ + "portality.models.background.BackgroundJob", # ~~-> BackgroundJob:Model~~ + "portality.models.autocheck.Autocheck" # ~~-> Autocheck:Model~~ ] # Map from dataobj coercion declarations to ES mappings @@ -1487,6 +1491,15 @@ "name": "Your group activity", "description": "Your dashboard shows you who is working on what, and the status of your group's applications" } + ], + "/admin/journal/*": [ + { + "roles": ["admin"], + "selectors": [".autochecks-manager-toggle"], + "content_id": "admin_journal_autochecks", + "name": "Autochecks", + "description": "Autochecks are available on some journals, and can help you to identify potential problems with the journal's metadata." + } ] } @@ -1529,3 +1542,12 @@ # worksheet name or tab name that datalog will write to DATALOG_JA_WORKSHEET_NAME = 'Added' + +################################################## +# Autocheck Resource configurations + +# Should we autocheck incoming applications and update requests +AUTOCHECK_INCOMING = False + +AUTOCHECK_RESOURCE_ISSN_ORG_TIMEOUT = 10 +AUTOCHECK_RESOURCE_ISSN_ORG_THROTTLE = 1 # seconds between requests diff --git a/portality/static/js/application_form.js b/portality/static/js/application_form.js index 3944845669..54fd313368 100644 --- a/portality/static/js/application_form.js +++ b/portality/static/js/application_form.js @@ -460,7 +460,6 @@ doaj.af.EditorialApplicationForm = class extends doaj.af.BaseApplicationForm { event.preventDefault(); let id = $(this).attr("data-id"); let type = $(this).attr("data-type"); - that.submitting = true; that.unlock({type : type, id : id}) }); @@ -477,7 +476,7 @@ doaj.af.EditorialApplicationForm = class extends doaj.af.BaseApplicationForm { // ignore any "view note" modal close button hits // FIXME: I don't love this, it feels brittle, but I don't have a better solution let exceptClasses = ["formulaic-notemodal-close"]; - let exceptIds = ["open_withdraw_reinstate"]; + let exceptIds = ["open_withdraw_reinstate", "unlock"]; let excepted = false; for (let i = 0; i < exceptClasses.length; i++) { if ($(event.currentTarget).hasClass(exceptClasses[i])) { diff --git a/portality/static/js/autochecks.js b/portality/static/js/autochecks.js new file mode 100644 index 0000000000..32da79de29 --- /dev/null +++ b/portality/static/js/autochecks.js @@ -0,0 +1,122 @@ +if (!window.hasOwnProperty("doaj")) { doaj = {}} +if (!doaj.hasOwnProperty("autocheckers")) { doaj.autocheckers = {}} + +doaj.autocheckers.ISSNActive = class { + + MESSAGES = { + "unable_to_access": "We were unable to access the ISSN.org service", + "not_found": "The ISSN ({{ISSN}}) was not found at ISSN.org", + "fully_validated": "The ISSN ({{ISSN}}) is fully registered at ISSN.org", + "not_validated": "The ISSN ({{ISSN}}) has not been registered at ISSN.org" + } + + ICONS = { + "unable_to_access": "clock", + "not_found": "x-circle", + "fully_validated": "check-circle", + "not_validated": "x-circle" + } + + STYLE = { + "unable_to_access": "error", + "not_found": "error", + "fully_validated": "success", + "not_validated": "warn" + } + + draw(autocheck) { + let icon = this.ICONS[autocheck.advice]; + let message = this.MESSAGES[autocheck.advice]; + message = message.replace("{{ISSN}}", autocheck.original_value); + + let style = this.STYLE[autocheck.advice]; + + let frag = `
+ ${message} (see record)
`; + return frag; + } +} + +doaj.autocheckers.KeepersRegistry = class { + MESSAGES = { + "missing": "Keepers does not show any content archived in {service}.", + "present": "The journal content is actively archived in {service}.", + "outdated": "The journal has content archived in {service} but it's not current.", + "not_recorded": "Keepers Registry does not currently record information about {service}." + } + + ICONS = { + "missing": "x-circle", + "present": "check-circle", + "outdated": "x-circle", + "not_recorded": "info" + } + + STYLE = { + "missing": "error", + "present": "success", + "outdated": "error", + "not_recorded": "info" + } + + draw(autocheck) { + let icon = this.ICONS[autocheck.advice]; + let message = this.MESSAGES[autocheck.advice]; + + let context = JSON.parse(autocheck.context); + message = message.replace("{service}", context.service); + + let style = this.STYLE[autocheck.advice]; + + let frag = `
+ ${message} (see record)
`; + return frag; + } +} + +doaj.autocheckers.registry = { + "issn_active": doaj.autocheckers.ISSNActive, + "keepers_registry": doaj.autocheckers.KeepersRegistry +} + +doaj.autocheckers.AutochecksManager = class { + constructor(params) { + this.selector = edges.getParam(params.selector, ".autochecks-manager") + + this.namespace = "autochecks-manager"; + + this.draw(); + } + + draw() { + if (!doaj.autochecks || !doaj.autochecks.checks) { + return; + } + + let d = new Date(doaj.autochecks.created_date); + let date = d.toLocaleDateString("en-GB", {year: "numeric", month: "long", day: "numeric"}); + + let toggleClass = edges.css_classes(this.namespace, "toggle"); + let frag = `Autochecks were made on ${date} Show All Autochecks`; + + $(this.selector).html(frag); + + let toggleSelector = edges.css_class_selector(this.namespace, "toggle"); + edges.on(toggleSelector, "click", this, "toggle"); + } + + toggle(element) { + let el = $(element); + let state = el.attr("data-state"); + let autocheckSelector = ".formulaic-autocheck-container"; + if (state === "shown") { + $(autocheckSelector).hide(); + el.attr("data-state", "hidden"); + el.html("Show All Autochecks") + } else { + $(autocheckSelector).show(); + el.attr("data-state", "shown"); + el.html("Hide All Autochecks"); + } + } +} \ No newline at end of file diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 075f35ac87..2e4722f1c0 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -278,7 +278,8 @@ $.extend(true, doaj, { 'on hold' : 'On Hold', 'ready' : 'Ready', 'rejected' : 'Rejected', - 'accepted' : 'Accepted' + 'accepted' : 'Accepted', + 'post_submission_review': "Autochecking", }, adminStatusMap: function(value) { @@ -3148,7 +3149,8 @@ $.extend(true, doaj, { "in progress" : "Under review by an editor", "completed" : "Under review by an editor", "on hold" : "Under review by an editor", - "ready" : "Under review by an editor" + "ready" : "Under review by an editor", + "post_submission_review": "Pending" }; this.draw = function () { @@ -3228,7 +3230,8 @@ $.extend(true, doaj, { var deleteLinkUrl = deleteLinkTemplate.replace("__application_id__", resultobj.id); var deleteClass = edges.css_classes(this.namespace, "delete", this); if (resultobj.es_type === "draft_application" || - resultobj.admin.application_status === "update_request") { + resultobj.admin.application_status === "update_request" || + resultobj.admin.application_status === "post_submission_review") { deleteLink = '
  • \ \ @@ -3520,7 +3523,9 @@ $.extend(true, doaj, { if (dir === undefined) { dir = ""; } - dir = " " + dir; + if (dir !== "") { + dir = " " + dir; + } sortOptions += ''; } diff --git a/portality/static/js/edges/publisher.update_requests.edge.js b/portality/static/js/edges/publisher.update_requests.edge.js index 958f7f8ece..b4d9a64ba6 100644 --- a/portality/static/js/edges/publisher.update_requests.edge.js +++ b/portality/static/js/edges/publisher.update_requests.edge.js @@ -17,7 +17,7 @@ $.extend(true, doaj, { result.link = doaj.publisherUpdatesSearchConfig.journalReadOnlyUrl + resultobj['id']; result.label = 'View'; - if (status === "update_request" || status === "revisions_required") { + if (status === "post_submission_review" || status === "update_request" || status === "revisions_required") { result.link = doaj.publisherUpdatesSearchConfig.journalUpdateUrl + resultobj.admin.current_journal; result.label = 'Edit'; } diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index c970e12c18..0c5932c3ce 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -581,6 +581,11 @@ var formulaic = { var selector = "." + this.containerClassTemplate.replace("{name}", name); return $(selector, context); }; + + this.widgetsContainer = function(params) { + var context = this.get_context(params); + return $("#" + params.name + "_widgets-container", context) + } }, edges : { @@ -1076,6 +1081,112 @@ var formulaic = { $(select2_elem).focus(); }, + _make_empty_container: function(namespace, containerName, form, fieldDef, additionalClasses, additionalStyles) { + if (!additionalClasses) { + additionalClasses = ""; + } + if (!additionalStyles) { + additionalStyles = ""; + } + + let containerSelector = edges.css_class_selector(namespace, containerName); + let elements = form.controlSelect.widgetsContainer({name: fieldDef.name}); + let existing = false; + for (let i = 0; i < elements.length; i++) { + let el = $(elements[i]); + let cont = el.find(containerSelector) + if (cont.length > 0) { + cont.empty(); + existing = true; + } + } + + if (!existing) { + let containerClass = edges.css_classes(namespace, containerName); + elements.append(`
    `); + } + + return elements.find(containerSelector); + }, + + newAutocheck : function(params) { + return edges.instantiate(formulaic.widgets.Autocheck, params); + }, + Autocheck: function(params) { + this.fieldDef = params.fieldDef; + this.form = params.formulaic; + + this.namespace = "formulaic-autocheck-" + this.fieldDef.name; + + this.init = function() { + let annos = this._getAutochecksForField(); + if (annos.length === 0) { + return; + } + + let generalClass = "formulaic-autocheck-container"; + let defaultDisplay = "style='display: none'"; + let cont = formulaic.widgets._make_empty_container(this.namespace, "autochecks", this.form, this.fieldDef, generalClass, defaultDisplay); + + let frag = ""; + for (let anno of annos) { + frag += this._renderAutocheck(anno) + } + + let listClass = edges.css_classes(this.namespace, "list") + frag = `
      ${frag}
    `; + cont.html(frag); + + feather.replace(); + } + + this._getAutochecksForField = function() { + if (!doaj.autochecks) { + return []; + } + let applicable = []; + for (let anno of doaj.autochecks.checks) { + if (anno.field && anno.field === this.fieldDef.name) { + applicable.push(anno); + } + } + return applicable; + } + + this._renderAutocheck = function(autocheck) { + let frag = "
  • "; + + if (autocheck.checked_by && doaj.autocheckers && + doaj.autocheckers.registry.hasOwnProperty(autocheck.checked_by)) { + frag += (new doaj.autocheckers.registry[autocheck.checked_by]()).draw(autocheck) + } else { + frag += this._defaultRender(autocheck); + } + + frag += `
    • `; + return frag; + } + + this._defaultRender = function(autocheck) { + let frag = ""; + if (autocheck.advice) { + frag += `${autocheck.advice}
    ` + } + if (autocheck.reference_url) { + frag += `    ${autocheck.reference_url}
    ` + } + if (autocheck.suggested_value) { + frag += `Suggested Value(s): ${autocheck.suggested_value.join(", ")}
    ` + } + if (autocheck.original_value) { + frag += `(Original value when automated checks ran: ${autocheck.original_value})` + } + return frag; + } + + this.init(); + }, + newSubjectTree : function(params) { return edges.instantiate(formulaic.widgets.SubjectTree, params); }, @@ -1202,7 +1313,7 @@ var formulaic = { this.fieldDef = params.fieldDef; this.form = params.formulaic; - this.ns = "formulaic-clickableowner"; + this.namespace = "formulaic-clickableowner"; this.link = false; @@ -1222,11 +1333,12 @@ var formulaic = { if (this.link) { this.link.attr("href", "/account/" + val); } else { - var classes = edges.css_classes(this.ns, "visit"); - var id = edges.css_id(this.ns, this.fieldDef.name); - that.after('

    See this account’s profile

    '); + let cont = formulaic.widgets._make_empty_container(this.namespace, "clickable_owner", this.form, this.fieldDef); + var classes = edges.css_classes(this.namespace, "visit"); + var id = edges.css_id(this.namespace, this.fieldDef.name); + cont.html('

    See this account’s profile

    '); - var selector = edges.css_id_selector(this.ns, this.fieldDef.name); + var selector = edges.css_id_selector(this.namespace, this.fieldDef.name); this.link = $(selector, this.form.context); } } else if (this.link) { @@ -1303,10 +1415,8 @@ var formulaic = { this.link = false; this.init = function() { - var elements = this.form.controlSelect.input( - {name: this.fieldDef.name}); - // TODO: should work as-you-type by changing "change" to "keyup" event; doesn't work in edges - //edges.on(elements, "change.ClickableUrl", this, "updateUrl"); + var elements = this.form.controlSelect.input({name: this.fieldDef.name}); + edges.on(elements, "keyup.ClickableUrl", this, "updateUrl"); for (var i = 0; i < elements.length; i++) { @@ -1317,16 +1427,17 @@ var formulaic = { this.updateUrl = function(element) { var that = $(element); var val = that.val(); - var id = edges.css_id(this.ns, this.fieldDef.name); if (val && (val.substring(0,7) === "http://" || val.substring(0,8) === "https://") && val.length > 10) { if (this.link) { - this.link.text(val); this.link.attr("href", val); } else { var classes = edges.css_classes(this.ns, "visit"); - that.after('

    ' + val + '

    '); - + var id = edges.css_id(this.ns, this.fieldDef.name); + that.after('

    \ + Open link\ + \ +

    '); var selector = edges.css_id_selector(this.ns, this.fieldDef.name); this.link = $(selector, this.form.context); } @@ -1376,9 +1487,10 @@ var formulaic = { if (this.container) { this.container.html('Full contents: ' + edges.escapeHtml(val) + ''); } else { + let cont = formulaic.widgets._make_empty_container(this.ns, "clickable_url", this.form, this.fieldDef); var classes = edges.css_classes(this.ns, "contents"); var id = edges.css_id(this.ns, this.fieldDef.name); - that.after('

    Full contents: ' + edges.escapeHtml(val) + '

    '); + cont.html('

    Full contents: ' + edges.escapeHtml(val) + '

    '); var selector = edges.css_id_selector(this.ns, this.fieldDef.name); this.container = $(selector, this.form.context); @@ -1619,7 +1731,7 @@ var formulaic = { let f = this.fields[idx]; let s2_input = $($(f).select2()); $(f).on("focus", formulaic.widgets._select2_shift_focus); - s2_input.after($('')); + s2_input.after($('')); if (idx !== 0) { s2_input.attr("required", false); s2_input.attr("data-parsley-validate-if-empty", "true"); @@ -1755,7 +1867,7 @@ var formulaic = { for (var idx = 0; idx < this.divs.length; idx++) { let div = $(this.divs[idx]); - div.append($('')); + div.append($('')); if (idx !== 0) { let inputs = div.find(":input"); diff --git a/portality/static/js/tourist.js b/portality/static/js/tourist.js index b2931f6dcf..19435dc786 100644 --- a/portality/static/js/tourist.js +++ b/portality/static/js/tourist.js @@ -9,22 +9,39 @@ doaj.tourist.init = function(params) { doaj.tourist.allTours = params.tours || []; doaj.tourist.cookiePrefix = params.cookie_prefix; - $(".trigger_tour").on("click", doaj.tourist.triggerTour); + let available = doaj.tourist.listAvailableTours(); + let availableIds = available.map((x) => { return x.content_id }); - let first = doaj.tourist.findNextTour(); - if (first) { - doaj.tourist.start(first); - } + let navContainer = $("#dropdown--tour_nav") + + if (availableIds.length === 0) { + navContainer.hide(); + } else { + let tourNav = $("#feature_tours li"); + for (let navEntry of tourNav) { + let ne = $(navEntry); + let trigger = ne.find("a.trigger_tour"); + let tourId = trigger.attr("data-tour-id"); + if (!availableIds.includes(tourId)) { + ne.hide(); + } + } - $("#dropdown--tour_nav").hoverIntent(doaj.tourist.showDropdown, doaj.tourist.hideDropdown); + $(".trigger_tour").on("click", doaj.tourist.triggerTour); + navContainer.show(); + navContainer.hoverIntent(doaj.tourist.showDropdown, doaj.tourist.hideDropdown); + + let first = doaj.tourist.findNextTour(); + if (first) { + doaj.tourist.start(first); + } + } } doaj.tourist.showDropdown = function(e) { - console.log("showDropdown` called") $("#feature_tours").show(); } doaj.tourist.hideDropdown = function() { - console.log("showDropdown` called") $("#feature_tours").hide(); } @@ -33,8 +50,25 @@ doaj.tourist.findNextTour = function() { let first = false; for (let tour of doaj.tourist.allTours) { + // check to see if required selectors are present on the page + if (tour.selectors) { + let selectorsPresent = 0; + for (let selector of tour.selectors) { + let el = $(selector); + if (el.length > 0) { + selectorsPresent++; + } + } + if (tour.selectors.length !== selectorsPresent) { + continue; + } + } + + // if we are good to go ahead, then check the cookies to see if the tour needs to be shown let cookieName = doaj.tourist.cookiePrefix + tour.content_id + "=" + tour.content_id; let cookie = cookies.find(c => c === cookieName); + + // if it has not previously been shown, then show it if (!cookie) { first = tour; break; @@ -43,6 +77,27 @@ doaj.tourist.findNextTour = function() { return first; } +doaj.tourist.listAvailableTours = function() { + let available = []; + for (let tour of doaj.tourist.allTours) { + // check to see if required selectors are present on the page + if (tour.selectors) { + let selectorsPresent = 0; + for (let selector of tour.selectors) { + let el = $(selector); + if (el.length > 0) { + selectorsPresent++; + } + } + if (tour.selectors.length !== selectorsPresent) { + continue; + } + } + available.push(tour); + } + return available; +} + doaj.tourist.start = function (tour) { doaj.tourist.contentId = tour.content_id; doaj.tourist.currentTour = new Tourguide({ diff --git a/portality/tasks/application_autochecks.py b/portality/tasks/application_autochecks.py new file mode 100644 index 0000000000..8566f31365 --- /dev/null +++ b/portality/tasks/application_autochecks.py @@ -0,0 +1,98 @@ +from portality import constants +from portality import models +from portality.background import BackgroundTask, BackgroundApi, BackgroundException +from portality.tasks.helpers import background_helper +from portality.tasks.redis_huey import main_queue +from portality.bll import DOAJ + + +class ApplicationAutochecks(BackgroundTask): + + __action__ = "application_autochecks" + + def run(self): + """ + Execute the task as specified by the background_jon + :return: + """ + job = self.background_job + params = job.params + app_id = self.get_param(params, "application", False) + status_on_complete = self.get_param(params, "status_on_complete", "") + + if app_id is False: + job.add_audit_message("Application ID must be provided when creating the job") + job.fail() + return + + application = models.Application.pull(app_id) + if application is None: + job.add_audit_message("Application id {id} is not present, no further action required".format(id=app_id)) + return + + job.add_audit_message("Running application autocheck for application with id {id}".format(id=app_id)) + + annoSvc = DOAJ.autochecksService() + annoSvc.autocheck_application(application, created_date=job.created_date, logger=lambda x: job.add_audit_message(x)) + + if status_on_complete != "" and status_on_complete in constants.APPLICATION_STATUSES_ALL: + job.add_audit_message("Setting status to {x}".format(x=status_on_complete)) + application.set_application_status(status_on_complete) + # Note: we have not locked the application + application.save() + + def cleanup(self): + """ + Cleanup after a successful OR failed run of the task + :return: + """ + pass + + @classmethod + def prepare(cls, username, **kwargs): + """ + Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, + or fail with a suitable exception + + :param kwargs: arbitrary keyword arguments pertaining to this task type + :return: a BackgroundJob instance representing this task + """ + + app_id = kwargs.get("application", False) + status_on_complete = kwargs.get("status_on_complete", "") + + if not app_id: + raise BackgroundException("'application' ID must be provided to prepare function") + + params = {} + cls.set_param(params, "application", app_id) + cls.set_param(params, "status_on_complete", status_on_complete) + + # first prepare a job record + job = background_helper.create_job(username=username, + action=cls.__action__, + params=params, + queue_id=huey_helper.queue_id, ) + + return job + + @classmethod + def submit(cls, background_job): + """ + Submit the specified BackgroundJob to the background queue + + :param background_job: the BackgroundJob instance + :return: + """ + background_job.save() + application_autochecks.schedule(args=(background_job.id,), delay=10) + + +huey_helper = ApplicationAutochecks.create_huey_helper(main_queue) + + +@huey_helper.register_execute(is_load_config=True) +def application_autochecks(job_id): + job = models.BackgroundJob.pull(job_id) + task = ApplicationAutochecks(job) + BackgroundApi.execute(task) diff --git a/portality/tasks/consumer_main_queue.py b/portality/tasks/consumer_main_queue.py index 308c2c3976..fa0b496617 100644 --- a/portality/tasks/consumer_main_queue.py +++ b/portality/tasks/consumer_main_queue.py @@ -2,6 +2,7 @@ # It changes the logging configuration. If it's imported anywhere else in the app, # it will change the logging configuration for the entire app. import logging +from portality.core import app logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) @@ -28,3 +29,7 @@ from portality.tasks.request_es_backup import scheduled_request_es_backup, request_es_backup # noqa from portality.tasks.find_discontinued_soon import scheduled_find_discontinued_soon, find_discontinued_soon # noqa from portality.tasks.datalog_journal_added_update import scheduled_datalog_journal_added_update, datalog_journal_added_update # noqa + +# Conditionally enable new application autochecking +if app.config.get("AUTOCHECK_INCOMING", False): + from portality.tasks.application_autochecks import application_autochecks diff --git a/portality/tasks/helpers/background_helper.py b/portality/tasks/helpers/background_helper.py index 6c59b875e5..847557e130 100644 --- a/portality/tasks/helpers/background_helper.py +++ b/portality/tasks/helpers/background_helper.py @@ -64,11 +64,13 @@ def execute_by_job_id(job_id, task_factory: TaskFactory): BackgroundApi.execute(task) -def execute_by_bg_task_type(bg_task_type: Type[BackgroundTask], **prepare_kwargs): +def execute_by_bg_task_type(bg_task_type: Type[BackgroundTask], job_wrapper=None, **prepare_kwargs): """ wrapper for execute by BackgroundTask """ user = app.config.get("SYSTEM_USERNAME") job = bg_task_type.prepare(user, **prepare_kwargs) + if job_wrapper is not None: + job = job_wrapper(job) task = bg_task_type(job) BackgroundApi.execute(task) diff --git a/portality/tasks/journal_autochecks.py b/portality/tasks/journal_autochecks.py new file mode 100644 index 0000000000..5403ac6b83 --- /dev/null +++ b/portality/tasks/journal_autochecks.py @@ -0,0 +1,91 @@ +from portality import models +from portality.background import BackgroundTask, BackgroundApi, BackgroundException +from portality.tasks.helpers import background_helper +from portality.tasks.redis_huey import long_running +from portality.bll import DOAJ + +####################################### +# NOTE: this background task is currently not in use, it is prepped for being used with +# autocheck-on-demand from the administrators via the user interface, which is not yet implemented +# +# When we implement that functionality we will also need to implement tests for this +####################################### + +class JournalAutochecks(BackgroundTask): + + __action__ = "journal_autochecks" + + def run(self): + """ + Execute the task as specified by the background_jon + :return: + """ + job = self.background_job + params = job.params + journal_id = self.get_param(params, "journal", False) + + if journal_id is False: + job.add_audit_message("Journal ID must be provided when creating the job") + job.fail() + return + + journal = models.Journal.pull(journal_id) + + job.add_audit_message("Running journal autocheck for journal with id {id}".format(id=journal_id)) + + annoSvc = DOAJ.autochecksService() + annoSvc.autocheck_journal(journal, logger=lambda x: job.add_audit_message(x)) + + def cleanup(self): + """ + Cleanup after a successful OR failed run of the task + :return: + """ + pass + + @classmethod + def prepare(cls, username, **kwargs): + """ + Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, + or fail with a suitable exception + + :param kwargs: arbitrary keyword arguments pertaining to this task type + :return: a BackgroundJob instance representing this task + """ + + journal_id = kwargs.get("journal", False) + + if not journal_id: + raise BackgroundException("'journal' ID must be provided to prepare function") + + params = {} + cls.set_param(params, "journal", journal_id) + + # first prepare a job record + job = background_helper.create_job(username=username, + action=cls.__action__, + params=params, + queue_id=huey_helper.queue_id, ) + + return job + + @classmethod + def submit(cls, background_job): + """ + Submit the specified BackgroundJob to the background queue + + :param background_job: the BackgroundJob instance + :return: + """ + background_job.save() + journal_autochecks.schedule(args=(background_job.id,), delay=10) + + +huey_helper = JournalAutochecks.create_huey_helper(long_running) + + +@huey_helper.register_execute(is_load_config=True) +def journal_autochecks(job_id): + job = models.BackgroundJob.pull(job_id) + task = JournalAutochecks(job) + BackgroundApi.execute(task) diff --git a/portality/templates/admin/article_metadata.html b/portality/templates/admin/article_metadata.html index 84866a9bc9..a8ec2d2f21 100644 --- a/portality/templates/admin/article_metadata.html +++ b/portality/templates/admin/article_metadata.html @@ -17,6 +17,8 @@

    {{ heading_object_type }} —
    {{form_context.source.bibjson().title}}

    + + See this article in DOAJ {% if form_context.source.current_journal %}

    View associated journal diff --git a/portality/templates/application_form/01-oa-compliance/index.html b/portality/templates/application_form/01-oa-compliance/index.html index 2341066f24..932e2b3953 100644 --- a/portality/templates/application_form/01-oa-compliance/index.html +++ b/portality/templates/application_form/01-oa-compliance/index.html @@ -1,6 +1,6 @@ {% if not notabs %}

    Page 1 of {{ FORM | length + 1}}

    {% endif %} -

    Open access compliance

    +
    Open access compliance
     {% if formulaic_context.name == "public" %}
    diff --git a/portality/templates/application_form/02-about/index.html b/portality/templates/application_form/02-about/index.html index 4a26977d79..afce05661b 100644 --- a/portality/templates/application_form/02-about/index.html +++ b/portality/templates/application_form/02-about/index.html @@ -3,7 +3,7 @@

    Page 2 of {{ FORM | length + 1}}

    {% endif %} -

    About the journal

    +
    About the journal
     {% set fs = formulaic_context.fieldset("about_the_journal") %} {% for f in fs.fields() %} diff --git a/portality/templates/application_form/03-copyright-licensing/index.html b/portality/templates/application_form/03-copyright-licensing/index.html index 4a7495c5c6..6bea0c4b44 100644 --- a/portality/templates/application_form/03-copyright-licensing/index.html +++ b/portality/templates/application_form/03-copyright-licensing/index.html @@ -3,7 +3,7 @@

    Page 3 of {{ FORM | length + 1}}

    {% endif %} -

    © Copyright & licensing

    +
    © Copyright & licensing
     {% set fs = formulaic_context.fieldset("licensing") %}

    {{ fs.label }}

    diff --git a/portality/templates/application_form/04-editorial/index.html b/portality/templates/application_form/04-editorial/index.html index ce52124044..e31a4913de 100644 --- a/portality/templates/application_form/04-editorial/index.html +++ b/portality/templates/application_form/04-editorial/index.html @@ -3,7 +3,7 @@

    Page 4 of {{ FORM | length + 1}}

    {% endif %} -

    Editorial

    +
    Editorial
     {% set fs = formulaic_context.fieldset("peer_review") %}

    {{ fs.label }}

    diff --git a/portality/templates/application_form/05-business-model/index.html b/portality/templates/application_form/05-business-model/index.html index 25e449e675..e3f2066b4e 100644 --- a/portality/templates/application_form/05-business-model/index.html +++ b/portality/templates/application_form/05-business-model/index.html @@ -3,7 +3,7 @@

    Page 5 of {{ FORM | length + 1}}

    {% endif %} -

    Business Model

    +
    Business Model
     {% set fs = formulaic_context.fieldset("apc") %}

    {{ fs.label }}

    {% for f in fs.fields() %} diff --git a/portality/templates/application_form/06-best-practice/index.html b/portality/templates/application_form/06-best-practice/index.html index 2c1d770df5..9cadc35164 100644 --- a/portality/templates/application_form/06-best-practice/index.html +++ b/portality/templates/application_form/06-best-practice/index.html @@ -3,7 +3,7 @@

    Page 6 of {{ FORM | length + 1}}

    {% endif %} -

    Best practice

    +
    Best practice

    The best practices in this section adhere to publishing standards based around findability, preserving the content and ethical publishing practices.

    We encourage journals to adopt these best practices but they are not mandatory for DOAJ indexing.

    diff --git a/portality/templates/application_form/07-review/index.html b/portality/templates/application_form/07-review/index.html index 3dd15fd8eb..ff73560741 100644 --- a/portality/templates/application_form/07-review/index.html +++ b/portality/templates/application_form/07-review/index.html @@ -5,9 +5,9 @@ {% set t = { "val": 0} %} {% if obj %} -

    {{ obj.data.bibjson.title }}

    +
    {{ obj.data.bibjson.title }}
     {% else %} -

    Review your answers

    +
    Review your answers
     {% endif %} diff --git a/portality/templates/application_form/_autochecks.html b/portality/templates/application_form/_autochecks.html new file mode 100644 index 0000000000..c3ce41c143 --- /dev/null +++ b/portality/templates/application_form/_autochecks.html @@ -0,0 +1,2 @@ +{# This div is populated by javascript class doaj.autocheckers.AutochecksManager #} +
    \ No newline at end of file diff --git a/portality/templates/application_form/_contact.html b/portality/templates/application_form/_contact.html index b1140a6e87..fd0f7c36b5 100644 --- a/portality/templates/application_form/_contact.html +++ b/portality/templates/application_form/_contact.html @@ -1,13 +1,16 @@ {% if obj.owner %} {% set account = obj.owner_account %} - {% if account.id == current_user.id %} -
      +
        + {% if account.id == current_user.id %}
      • Assigned to you
        • -
      - {% else %} - - {% endif %} + {% endif %} + {% if obj.application_status %} +
    • + Status: {{ obj.application_status }} +
      • + {% endif %} +
    {% endif %} \ No newline at end of file diff --git a/portality/templates/application_form/_field.html b/portality/templates/application_form/_field.html index 2e6ce601a5..0e1d37e1d5 100644 --- a/portality/templates/application_form/_field.html +++ b/portality/templates/application_form/_field.html @@ -21,7 +21,7 @@ {% endif %} {% if f.help("short_help") %} -

    {{ f.help("short_help") | safe }}

    +

    {{ f.help("short_help") | safe }}

    {% endif %} {% if f.get("hint") %}

    {{ f.hint | safe }}

    {% endif %} @@ -65,6 +65,10 @@ {% endif %} + {% if f.widgets %} +
    + {% endif %} + {% include "application_form/modal.html" %} diff --git a/portality/templates/application_form/editorial_form_body.html b/portality/templates/application_form/editorial_form_body.html index 53be1f5437..9750c1ceff 100644 --- a/portality/templates/application_form/editorial_form_body.html +++ b/portality/templates/application_form/editorial_form_body.html @@ -1,8 +1,13 @@ {% include "application_form/_edit_status.html" %} {% include "application_form/_backend_validation.html" %} +{% include "application_form/_autochecks.html" %} {% import "application_form/_application_warning_msg.html" as _msg %} -{{ _msg.build_journal_withdrawn_deleted_msg(obj) }} +{% if obj and (obj.es_type == 'journal' and obj.is_in_doaj()) %} + {{ _msg.build_journal_withdrawn_deleted_msg(obj) }} + See this journal in DOAJ +{% endif %} + Administrative -{% if quick_reject and obj.application_status not in ["accepted", "rejected"] and obj.current_journal == None %} - -{% endif %} - -{% if withdrawable %} - {% set withdraw_reinstate = "Withdraw" if obj.is_in_doaj() else "Reinstate" %} - {% if job %} - {% set withdraw_reinstate = "Withdrawing" if obj.is_in_doaj() else "Reinstating" %} - {% endif %} - -{% endif %} - -{% set fs = formulaic_context.fieldset("reviewers") %} -{% if fs %} -
    -

    {{ fs.label }}

    -
    - {% for f in fs.fields() %} -
    - {% set field_template = f.template %} - {% include field_template %} -
    - {% endfor %} -
    -
    -{% endif %} - -{% if formulaic_context.fieldset("reassign") or formulaic_context.fieldset("status") %}
    -
    - {% set fs = formulaic_context.fieldset("reassign") %} +
    Administrative
     + {% set fs = formulaic_context.fieldset("reviewers") %} {% if fs %} - {% for f in fs.fields() %} -
    -

    {{ fs.label }}

    +

    {{ fs.label }}

    +
    + {% for f in fs.fields() %} +
    {% set field_template = f.template %} {% include field_template %} -
    - {% endfor %} +
    + {% endfor %} +
    {% endif %} - {% set fs = formulaic_context.fieldset("status") %} - {% if fs %} - {% for f in fs.fields() %} -
    -

    {{ fs.label }}

    + {% if formulaic_context.fieldset("reassign") or formulaic_context.fieldset("status") %} + {% set fs = formulaic_context.fieldset("reassign") %} + {% if fs %} + {% for f in fs.fields() %} +

    {{ fs.label }}

    + {% set field_template = f.template %} + {% include field_template %} + {% endfor %} + {% endif %} + + {% set fs = formulaic_context.fieldset("status") %} + {% if fs %} + {% for f in fs.fields() %} +

    {{ fs.label }}

    {% set field_template = f.template %} {% include field_template %} -
    - {% endfor %} + {% endfor %} + {% endif %} {% endif %} -
    -
    -{% endif %} -{% set fs = formulaic_context.fieldset("continuations") %} -{% if fs %} -
    -

    {{ fs.label }}

    -
    + {% set fs = formulaic_context.fieldset("continuations") %} + {% if fs %} +

    {{ fs.label }}

    {% for f in fs.fields() %} -
    {% set field_template = f.template %} - {% include field_template %} -
    - {% if loop.index0 % 2 == 1 %} -
    - {% endif %} + {% include field_template %} {% endfor %} -
    -
    -{% endif %} + {% endif %} -
    -
    -
    - {% set fs = formulaic_context.fieldset("subject") %} - {% if fs %} -

    {{ fs.label }}

    - {% for f in fs.fields() %} - {% set field_template = f.template %} - {% include field_template %} - {% endfor %} -

    Selected:

    -
    - {% endif %} -
    + {% set fs = formulaic_context.fieldset("subject") %} + {% if fs %} +

    {{ fs.label }}

    + {% for f in fs.fields() %} + {% set field_template = f.template %} + {% include field_template %} + {% endfor %} +

    Selected:

    +
    + {% endif %} -
    - {% set fs = formulaic_context.fieldset("seal") %} - {% if fs %} -

    {{ fs.label }}

    -

    The journal has fulfilled all the criteria for the Seal.

    - {% for f in fs.fields() %} - {% set field_template = f.template %} - {% include field_template %} - {% endfor %} - {% endif %} -
    -
    + {% set fs = formulaic_context.fieldset("seal") %} + {% if fs %} +

    {{ fs.label }}

    +

    The journal may have fulfilled all the criteria for the Seal.

    + {% for f in fs.fields() %} + {% set field_template = f.template %} + {% include field_template %} + {% endfor %} + {% endif %}
    diff --git a/portality/templates/application_form/editorial_side_panel.html b/portality/templates/application_form/editorial_side_panel.html index 34db9ba93e..9ddf55698f 100644 --- a/portality/templates/application_form/editorial_side_panel.html +++ b/portality/templates/application_form/editorial_side_panel.html @@ -1,30 +1,62 @@ -{% include "application_form/_contact.html" %} -{% if obj %} -

    LOCKED FOR EDITING UNTIL {{lock.expire_formatted()}}

    - {% if obj.application_status != constants.APPLICATION_STATUS_ACCEPTED %} - - - {% endif %} - - -{% endif %} +
    + {% include "application_form/_contact.html" %} + {% if obj %} +

    Locked for editing until {{ lock.expire_formatted() }} +

    +
    + {% if obj.application_status != constants.APPLICATION_STATUS_ACCEPTED %} + + + {% endif %} + + +
    + {% endif %} -{% set fs = formulaic_context.fieldset("optional_validation") %} -{% if fs %} - {% for f in fs.fields() %} - {% set field_template = f.template %} - {% include field_template %} - {% endfor %} -{% endif %} + {% if withdrawable %} +
    + {% set withdraw_reinstate = "Withdraw" if obj.is_in_doaj() else "Reinstate" %} + {% if job %} + {% set withdraw_reinstate = "Withdrawing" if obj.is_in_doaj() else "Reinstating" %} + {% endif %} + +
    + {% endif %} - + {% if quick_reject and obj.application_status not in ["accepted", "rejected"] and obj.current_journal == None %} +
    + +
    + {% endif %} + + {% set fs = formulaic_context.fieldset("optional_validation") %} + {% if fs %} + {% for f in fs.fields() %} + {% set field_template = f.template %} + {% include field_template %} + {% endfor %} + {% endif %} + + +
    diff --git a/portality/templates/application_form/maned_application.html b/portality/templates/application_form/maned_application.html index 257b3f3cdd..1bb6faf5b3 100644 --- a/portality/templates/application_form/maned_application.html +++ b/portality/templates/application_form/maned_application.html @@ -42,5 +42,17 @@

    {% endblock %} {% block extra_js_bottom scoped %} + {% if autochecks %} + + + {% endif %} {% include "application_form/js/_form_js.html" %} + {% if autochecks %} + + {% endif %} {% endblock %} diff --git a/portality/templates/application_form/maned_journal.html b/portality/templates/application_form/maned_journal.html index 2912311521..fdc129a06d 100644 --- a/portality/templates/application_form/maned_journal.html +++ b/portality/templates/application_form/maned_journal.html @@ -87,6 +87,18 @@

    {% endblock %} {% block extra_js_bottom scoped %} + {% if autochecks %} + + + {% endif %} {% set factory = "journal" %} {% include "application_form/js/_form_js.html" %} + {% if autochecks %} + + {% endif %} {% endblock %} diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html index 5206cb0cab..069ba89633 100644 --- a/portality/templates/dashboard/_todo.html +++ b/portality/templates/dashboard/_todo.html @@ -29,6 +29,18 @@ "feather": "inbox", "link" : url_for('admin.suggestions') + "?source=" + search_query_source(term=[{"admin.application_status.exact":"pending"}]) }, + constants.TODO_MANED_LAST_MONTH_UPDATE_REQUEST: { + "text" : "Last month's Update Request Process Update", + "colour" : "var(--grapefruit)", + "feather": "edit", + "show_status": true + }, + constants.TODO_MANED_NEW_UPDATE_REQUEST: { + "text" : "Recent Update Request Process Update", + "colour" : "var(--grapefruit)", + "feather": "edit", + "show_status": true + }, constants.TODO_EDITOR_STALLED: { "text" : "Stalled Chase Associate Editor", "show_status": true, @@ -88,6 +100,7 @@ %}
    + {% if todos|length == 0 %}

    diff --git a/portality/templates/dashboard/index.html b/portality/templates/dashboard/index.html index 79bf1c26e9..99459988bd 100644 --- a/portality/templates/dashboard/index.html +++ b/portality/templates/dashboard/index.html @@ -3,6 +3,25 @@ {% block content %} +

    + {% if request.values.get("filter") == None %} + Show all + {% else %} + Show all + {% endif %} + + {% if request.values.get("filter") == "na" %} + New Applications + {% else %} + New Applications + {% endif %} + + {% if request.values.get("filter") == "ur" %} + Update Requests + {% else %} + Update Requests + {% endif %} +
    {% include "dashboard/_todo.html" %}
    {# ~~->$GroupStatus:Feature~~ #} diff --git a/portality/templates/doaj/article.html b/portality/templates/doaj/article.html index 91f3f80200..b1fc9caf4a 100644 --- a/portality/templates/doaj/article.html +++ b/portality/templates/doaj/article.html @@ -61,6 +61,7 @@ %} {% set bibjson = article.bibjson() %} + {% set id = article.id %} {% set jbib = journal.bibjson() %} {% set jtitle, cite = bibjson.vancouver_citation() %} {% set issns = bibjson.issns() %} @@ -70,6 +71,9 @@
    + {% if not current_user.is_anonymous and current_user.has_role("admin") %} + Edit this article + {% endif %}

    {{jtitle}} ({{ bibjson.get_publication_date(date_format="%b %Y") }}) diff --git a/portality/templates/doaj/index.html b/portality/templates/doaj/index.html index 1ab4015af5..5862468c45 100644 --- a/portality/templates/doaj/index.html +++ b/portality/templates/doaj/index.html @@ -243,6 +243,7 @@

    Recently-added journals

    DOAJ’s team of managing editors, editors, and volunteers work with publishers to index new journals. As soon as they’re accepted, these journals are displayed on our website freely accessible to everyone.

    {# TODO: Make sure these links work #}

    → See Atom feed

    +

    → A log of journals added (and withdrawn)

    → DOWNLOAD all journals as CSV

    diff --git a/portality/templates/doaj/toc.html b/portality/templates/doaj/toc.html index e4672fb8dd..e1ad8c689c 100644 --- a/portality/templates/doaj/toc.html +++ b/portality/templates/doaj/toc.html @@ -273,6 +273,11 @@

    Best practice

    target="_blank" rel="noopener"> {{ policy }} + {% elif bibjson.deposit_policy_url %} + + {{ policy }} + {% else %} {{ policy }} {% endif %} diff --git a/portality/templates/includes/_tourist_nav.html b/portality/templates/includes/_tourist_nav.html index ef8d33090a..ab990f8b29 100644 --- a/portality/templates/includes/_tourist_nav.html +++ b/portality/templates/includes/_tourist_nav.html @@ -2,7 +2,7 @@ {% set tours = tourservice.activeTours(request.path, current_user) %} {% if tours|length > 0 %} -
  • +

    • {# this next bit has to be all on one line so that the spacing is correct #} - {% if bibjson.pissn %}{{bibjson.pissn}} (Print){% endif %}{% if bibjson.eissn %}{% if bibjson.pissn %} / {% endif %}{{bibjson.eissn}} (Online){% endif %} + {% if bibjson.pissn %}{{ bibjson.pissn }} (Print){% endif %}{% if bibjson.eissn %} + {% if bibjson.pissn %} / {% endif %}{{ bibjson.eissn }} (Online){% endif %}

    @@ -60,7 +64,8 @@

    {% if p.is_in_doaj() %} {{ bibjson.title }} {% else %} - {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in DOAJ) + {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in + DOAJ) {% endif %} {% endif %} {% if not loop.last %}; {% endif %} @@ -77,7 +82,8 @@

    {% if f.is_in_doaj() %} {{ bibjson.title }} {% else %} - {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in DOAJ) + {{ bibjson.title }}, ISSN: {{ bibjson.get_preferred_issn() }} (not available in + DOAJ) {% endif %} {% endif %} {% if not loop.last %}; {% endif %} @@ -87,21 +93,21 @@

    @@ -109,16 +115,16 @@

    @@ -25,6 +25,17 @@

    Explanation of XML errors

    + + + + +
    + Text decode failed, expected utf-8 encoded XML + + We were not able to read the file that you provided and it isn't in a recognised format. It may not be XML. + + You can only upload XML files only to us and they must be in one of the formats we accept: DOAJ or Crossref XML. +
    X articles imported ([number] new, [number] updated); N articles failed @@ -141,10 +152,10 @@

    Explanation of XML errors

    Unable to validate document with identified schema     		- No XML schema could be found. The schema, in particular DOAJ’s standard schema, is necessary to read an XML document. + No XML schema could be found. The schema, in particular DOAJ’s standard schema, is necessary to read an XML document. This message also dislpays when ORCID IDs are not formatted correctly. - Check that the file conforms to the DOAJ schema using a validator.
    + Check that the file conforms to the DOAJ schema using a validator. In particular, make sure that ORCID URLs start with https and not http.

    Often, the problem arises because the XML is missing a required tag. The schema validation will fail if you try to upload XML to DOAJ that is missing a specific tag. You can see exactly which tags are required here.

    diff --git a/portality/templates/publisher/nav.html b/portality/templates/publisher/nav.html index 2046ae2f16..25959d6f90 100644 --- a/portality/templates/publisher/nav.html +++ b/portality/templates/publisher/nav.html @@ -8,14 +8,14 @@ {% set csv = url_for('publisher.journal_csv') %} {% set tabs = [ - (index, "My drafts", 0, constants.ROLE_PUBLISHER), + (index, "My draft applications", 0, constants.ROLE_PUBLISHER), (journals, "My journals", 1, constants.ROLE_PUBLISHER), (urs, "My update requests", 2, constants.ROLE_PUBLISHER), (xml, "Upload article XML", 3, constants.ROLE_PUBLISHER), (metadata, "Enter article metadata", 4, constants.ROLE_PUBLISHER), (preservation, "Upload preservation file", 5, constants.ROLE_PUBLISHER_PRESERVATION), (csv, "Validate your CSV", 6, constants.ROLE_PUBLISHER_JOURNAL_CSV), - (help, "Help", 7, constants.ROLE_PUBLISHER), + (help, "XML help", 7, constants.ROLE_PUBLISHER), ] %} diff --git a/portality/templates/testdrive/testdrive.html b/portality/templates/testdrive/testdrive.html new file mode 100644 index 0000000000..2ae6df01ad --- /dev/null +++ b/portality/templates/testdrive/testdrive.html @@ -0,0 +1,46 @@ + + + + + {{ name }} - DOAJ Testdrive + + + + + +

    {{ name }} - Testdrive setup results

    + +{% for k, v in params.items() %} + {% if k != "teardown" %} +

    {{ k }}

    +
    + {% if v is mapping %} + {% for k1, v1 in v.items() %} +
    {{ k1 }}
    +
    + {% if v1 is iterable and (var is not string and var is not mapping) %} + {{ v1 }} + {% else %} + {% if v1.startswith("http") %}{% endif %}{{ v1 }}{% if v1.startswith("http") %}{% endif %}
    + {% endif %} + + {% endfor %} + {% elif v is iterable %} +
    list of values:
    + {% for v1 in v %} +
    + {% if v1.startswith("http") %}{% endif %}{{ v1 }}{% if v1.startswith("http") %}{% endif %}
    + {% endfor %} + {% endif %} +
    + {% endif %} +{% endfor %} + +

    Teardown

    + +

    When you have finished your test, you can cleanup the test resources from the test system by clicking this link

    + +{{ params.teardown }} + + + \ No newline at end of file diff --git a/portality/ui/messages.py b/portality/ui/messages.py index 35fca1675d..e7ba80ef7b 100644 --- a/portality/ui/messages.py +++ b/portality/ui/messages.py @@ -7,9 +7,9 @@ class Messages(object): PUBLISHER_APPLICATION_UPDATE_SUBMITTED_FLASH = (""" Your update request has been submitted. You may make further changes until the DOAJ Editorial Team picks it up - for review. Click the 'Edit' button to make further changes, or 'Delete' to cancel the request. + for review. Click the 'Edit' button to make further changes or 'Delete' to cancel the request. """, 'success') - PUBLISHER_UPLOAD_ERROR = """An error has occurred and your upload may not have succeeded. {error_str}
    If the problem persists please report the issue with the ID: {id}""" + PUBLISHER_UPLOAD_ERROR = """An error has occurred and your upload may not have succeeded. {error_str}
    If the problem persists, please send the error details from the Notes column of the History of Uploads box below and a screenshot of the entire error message""" NO_FILE_UPLOAD_ID="""No file upload record has been specified""" ARTICLE_METADATA_SUBMITTED_FLASH = ("Article created/updated", "success") @@ -50,7 +50,7 @@ class Messages(object): EXCEPTION_ARTICLE_BATCH_DUPLICATE = "One or more articles in this batch have duplicate identifiers" EXCEPTION_ARTICLE_BATCH_FAIL = "One or more articles failed to ingest; entire batch ingest halted" EXCEPTION_ARTICLE_BATCH_CONFLICT = "One or more articles in this batch matched multiple articles as duplicates; entire batch ingest halted" - EXCEPTION_DETECT_DUPLICATE_NO_ID = "The article you provided has neither doi nor fulltext url, and as a result cannot be deduplicated" + EXCEPTION_DETECT_DUPLICATE_NO_ID = "The article you provided has neither doi nor fulltext URL, and as a result cannot be deduplicated" EXCEPTION_ARTICLE_MERGE_CONFLICT = "The article matched multiple existing articles as duplicates, and we cannot tell which one to update" EXCEPTION_NO_DOI_NO_FULLTEXT = "The article must have a DOI and/or a Full-Text URL" EXCEPTION_ARTICLE_OVERRIDE = "Cannot update the article. An article with this URL and DOI already exists. If you are sure you want to replace it please delete it and then re-create it." @@ -63,13 +63,13 @@ class Messages(object): EXCEPTION_TOO_MANY_ISSNS = "Too many ISSNs. Only 2 ISSNs are allowed: one Print ISSN and one Online ISSN." EXCEPTION_MISMATCHED_ISSNS = "ISSNs provided don't match any journal." EXCEPTION_ISSNS_OF_THE_SAME_TYPE = "Both ISSNs have the same type: {type}" - EXCEPTION_IDENTICAL_PISSN_AND_EISSN = "The Print and Online ISSNs supplied are identical. If you supply 2 ISSNs they must be different." - EXCEPTION_NO_ISSNS = "Neither Print ISSN nor Online ISSN has been supplied. DOAJ requires at least one ISSN." + EXCEPTION_IDENTICAL_PISSN_AND_EISSN = "The Print and Online ISSNs supplied are identical. If you supply two ISSNs, they must be different." + EXCEPTION_NO_ISSNS = "Neither the Print ISSN nor Online ISSN have been supplied. DOAJ requires at least one ISSN." EXCEPTION_INVALID_BIBJSON = "Invalid article bibjson: " # + Dataobj exception message EXCEPTION_IDENTIFIER_CHANGE_CLASH = "DOI or Fulltext URL has been changed to match another article that already exists in DOAJ" - EXCEPTION_IDENTIFIER_CHANGE = "DOI or Fulltext URL have been changed. This operation is not permitted, please contact an administrator for help." - EXCEPTION_DUPLICATE_NO_PERMISSION = "You do not have the permissions to carry out the requested change" + EXCEPTION_IDENTIFIER_CHANGE = "Either the DOI or Fulltext URL has been changed. This operation is not permitted; please contact an administrator for help." + EXCEPTION_DUPLICATE_NO_PERMISSION = "You do not have permission to carry out the requested change" EXCEPTION_EDITING_ACCEPTED_JOURNAL = "You cannot edit applications which have been accepted into DOAJ." EXCEPTION_EDITING_WITHDRAWN_JOURNAL = "This journal has been withdrawn, update request cannot be accepted." @@ -84,7 +84,7 @@ class Messages(object): If you would like to see more results, you can download all of our data from {data_dump_url}. You can also harvest from our OAI-PMH endpoints; articles: {oai_article_url}, journals: {oai_journal_url}""" - CONSENT_COOKIE_VALUE = """By using the DOAJ website you have agreed to our cookie policy.""" + CONSENT_COOKIE_VALUE = """By using our website, you have agreed to our cookie policy.""" FORMS__APPLICATION_PROCESSORS__NEW_APPLICATION__FINALISE__USER_EMAIL_ERROR = "We were unable to send you an email confirmation - possible problem with the email address provided" FORMS__APPLICATION_PROCESSORS__NEW_APPLICATION__FINALISE__LOG_EMAIL_ERROR = 'Error sending application received email.' @@ -116,7 +116,18 @@ class Messages(object): DISCONTINUED_JOURNALS_FOUND_NOTIFICATION_ERROR_LOG = "Error sending notification with journals discontinuing soon." NO_DISCONTINUED_JOURNALS_FOUND_LOG = "No journals discontinuing soon found" - JOURNAL_CSV_VALIDATE__HEADER_CASE_MISMATCH = '"{h}" has mismatching case to expected header "{expected}".' + FORMS__APPLICATION_STATUS__PENDING = "Pending" + FORMS__APPLICATION_STATUS__IN_PROGRESS = 'In Progress' + FORMS__APPLICATION_STATUS__COMPLETED = 'Completed' + FORMS__APPLICATION_STATUS__POST_SUBMISSION_REVIEW = 'Post Submission Automation' + FORMS__APPLICATION_STATUS__UPDATE_REQUEST = 'Update Request' + FORMS__APPLICATION_STATUS__REVISIONS_REQUIRED = 'Revisions Required' + FORMS__APPLICATION_STATUS__ON_HOLD = 'On Hold' + FORMS__APPLICATION_STATUS__READY = 'Ready' + FORMS__APPLICATION_STATUS__REJECTED = 'Rejected' + FORMS__APPLICATION_STATUS__ACCEPTED = 'Accepted' + + JOURNAL_CSV_VALIDATE__HEADER_CASE_MISMATCH = '"{h}" has a mismatching case to the expected header "{expected}".' JOURNAL_CSV_VALIDATE__INVALID_HEADER = '"{h}" is not a valid column header. Please revert it to match what was sent to you in the original file.' JOURNAL_CSV_VALIDATE__REQUIRED_HEADER_MISSING = '"{h}" is a required column missing from this upload. Please refer to the original file and restore the column.' JOURNAL_CSV_VALIDATE__MISSING_JOURNAL = "There is no journal record in DOAJ for ISSN(s) {issns}. The record may not exist, or it may be withdrawn." @@ -126,6 +137,8 @@ class Messages(object): JOURNAL_CSV_VALIDATE__CANNOT_MAKE_UR = "We couldn't create an update for this journal because: {reason}" JOURNAL_CSV_VALIDATE__INVALID_DATA = "We couldn't understand the information in '{question}'" + PRESERVATION_NO_FILE = "No file provided for upload" + @classmethod def flash(cls, tup): if isinstance(tup, tuple): diff --git a/portality/view/admin.py b/portality/view/admin.py index 71bfaafb12..484441a02a 100644 --- a/portality/view/admin.py +++ b/portality/view/admin.py @@ -215,6 +215,8 @@ def journal_page(journal_id): return render_template("admin/journal_locked.html", journal=journal, lock=l.lock) fc = JournalFormFactory.context("admin", extra_param=exparam_editing_user()) + autochecks = models.Autocheck.for_journal(journal_id) + if request.method == "GET": job = None job_id = request.values.get("job") @@ -233,7 +235,7 @@ def journal_page(journal_id): return fc.render_template(lock=lockinfo, job=job, obj=journal, lcc_tree=lcc_jstree, past_cont_list=past_cont_list, future_cont_list=future_cont_list, - ) + autochecks=autochecks) elif request.method == "POST": processor = fc.processor(formdata=request.form, source=journal) @@ -248,7 +250,7 @@ def journal_page(journal_id): flash(str(e)) return redirect(url_for("admin.journal_page", journal_id=journal.id, _anchor='cannot_edit')) else: - return fc.render_template(lock=lockinfo, obj=journal, lcc_tree=lcc_jstree) + return fc.render_template(lock=lockinfo, obj=journal, lcc_tree=lcc_jstree, autochecks=autochecks) DisplayContData = namedtuple('DisplayContData', ['issn', 'title', 'id']) @@ -405,10 +407,12 @@ def application(application_id): fc = ApplicationFormFactory.context("admin", extra_param=exparam_editing_user()) form_diff, current_journal = ApplicationFormXWalk.update_request_diff(ap) + autochecks = models.Autocheck.for_application(application_id) + if request.method == "GET": fc.processor(source=ap) return fc.render_template(obj=ap, lock=lockinfo, form_diff=form_diff, - current_journal=current_journal, lcc_tree=lcc_jstree) + current_journal=current_journal, lcc_tree=lcc_jstree, autochecks=autochecks) elif request.method == "POST": processor = fc.processor(formdata=request.form, source=ap) @@ -428,8 +432,7 @@ def application(application_id): flash(str(e)) return redirect(url_for("admin.application", application_id=ap.id, _anchor='cannot_edit')) else: - return fc.render_template(obj=ap, lock=lockinfo, form_diff=form_diff, current_journal=current_journal, - lcc_tree=lcc_jstree) + return fc.render_template(obj=ap, lock=lockinfo, form_diff=form_diff, current_journal=current_journal, lcc_tree=lcc_jstree, autochecks=autochecks) @blueprint.route("/application_quick_reject/", methods=["POST"]) diff --git a/portality/view/dashboard.py b/portality/view/dashboard.py index de09b47455..2f63949a1b 100644 --- a/portality/view/dashboard.py +++ b/portality/view/dashboard.py @@ -1,5 +1,5 @@ from flask import Blueprint, make_response -from flask import render_template, abort +from flask import render_template, abort, request from flask_login import current_user, login_required from portality.decorators import ssl_required @@ -18,9 +18,19 @@ @login_required @ssl_required def top_todo(): + filter = request.values.get("filter") + new_applications, update_requests = True, True + if filter == "na": + update_requests = False + elif filter == "ur": + new_applications = False + # ~~-> Todo:Service~~ svc = DOAJ.todoService() - todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE")) + todos = svc.top_todo(current_user._get_current_object(), + size=app.config.get("TODO_LIST_SIZE"), + new_applications=new_applications, + update_requests=update_requests) # ~~-> Dashboard:Page~~ return render_template('dashboard/index.html', todos=todos) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 649eab8478..a10217534c 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -342,29 +342,17 @@ def toc(identifier=None): @blueprint.route("/toc/articles/") -@blueprint.route("/toc/articles//") -@blueprint.route("/toc/articles///") -def toc_articles(identifier=None, volume=None, issue=None): +def toc_articles_legacy(identifier=None): + return redirect(url_for('doaj.toc_articles', identifier=identifier, volume=1, issue=1), 301) + +@blueprint.route("/toc//articles") +def toc_articles(identifier=None): journal = find_toc_journal_by_identifier(identifier) bibjson = journal.bibjson() real_identifier = find_correct_redirect_identifier(identifier, bibjson) if real_identifier: - return redirect(url_for('doaj.toc_articles', identifier=real_identifier, - volume=volume, issue=issue), 301) + return redirect(url_for('doaj.toc_articles', identifier=real_identifier), 301) else: - - if is_issn_by_identifier(identifier) and volume: - filters = [dao.Facetview2.make_term_filter('bibjson.journal.volume.exact', volume)] - if issue: - filters += [dao.Facetview2.make_term_filter('bibjson.journal.number.exact', issue)] - q = dao.Facetview2.make_query(filters=filters) - - # The issn we are using to build the TOC - issn = bibjson.get_preferred_issn() - return redirect(url_for('doaj.toc', identifier=issn) - + '?source=' + dao.Facetview2.url_encode_query(q)) - - # now render all that information return render_template('doaj/toc_articles.html', journal=journal, bibjson=bibjson ) diff --git a/portality/view/doajservices.py b/portality/view/doajservices.py index 7d67cc03de..48c9c65003 100644 --- a/portality/view/doajservices.py +++ b/portality/view/doajservices.py @@ -112,3 +112,29 @@ def group_status(group_id): svc = DOAJ.todoService() stats = svc.group_stats(group_id) return make_response(json.dumps(stats)) + + +@blueprint.route("/autocheck/dismiss//", methods=["GET", "POST"]) +@jsonp +@login_required +def dismiss_autocheck(autocheck_set_id, autocheck_id): + if not current_user.has_role("admin"): + abort(404) + svc = DOAJ.autochecksService() + done = svc.dismiss(autocheck_set_id, autocheck_id) + if not done: + abort(404) + return make_response(json.dumps({"status": "success"})) + +@blueprint.route("/autocheck/undismiss//", methods=["GET", "POST"]) +@jsonp +@login_required +def undismiss_autocheck(autocheck_set_id, autocheck_id): + if not current_user.has_role("admin"): + abort(404) + svc = DOAJ.autochecksService() + done = svc.undismiss(autocheck_set_id, autocheck_id) + if not done: + abort(404) + return make_response(json.dumps({"status": "success"})) + diff --git a/portality/view/editor.py b/portality/view/editor.py index 24e28d8a26..0ee5ed5118 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -29,7 +29,7 @@ def restrict(): def index(): # ~~-> Todo:Service~~ svc = DOAJ.todoService() - todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE")) + todos = svc.top_todo(current_user._get_current_object(), size=app.config.get("TODO_LIST_SIZE"), update_requests=False) # ~~-> Dashboard:Page~~ return render_template('editor/dashboard.html', todos=todos) diff --git a/portality/view/publisher.py b/portality/view/publisher.py index 86cd10bfdc..43ef3d7d97 100644 --- a/portality/view/publisher.py +++ b/portality/view/publisher.py @@ -212,9 +212,8 @@ def upload_file(): job = IngestArticlesBackgroundTask.prepare(current_user.id, upload_file=f, schema=schema, url=url, previous=previous) IngestArticlesBackgroundTask.submit(job) except TaskException as e: - magic = str(uuid.uuid1()) - flash(Messages.PUBLISHER_UPLOAD_ERROR.format(error_str="", id=magic)) - app.logger.exception('File upload error. ' + magic) + flash(Messages.PUBLISHER_UPLOAD_ERROR.format(error_str=str(e))) + app.logger.exception('File upload error. ' + str(e)) return resp except BackgroundException as e: if str(e) == Messages.NO_FILE_UPLOAD_ID: @@ -223,10 +222,8 @@ def upload_file(): schema = "" return render_template('publisher/uploadmetadata.html', previous=previous, schema=schema, error=True) - - magic = str(uuid.uuid1()) - flash(Messages.PUBLISHER_UPLOAD_ERROR.format(error_str=str(e), id=magic)) - app.logger.exception('File upload error. ' + magic + '; ' + str(e)) + flash(Messages.PUBLISHER_UPLOAD_ERROR.format(error_str=str(e))) + app.logger.exception('File upload error. ' + str(e)) return resp if f is not None and f.filename != "": @@ -266,25 +263,29 @@ def preservation(): if request.method == "POST": f = request.files.get("file") - app.logger.info(f"Preservation file {f.filename}") + resp = make_response(redirect(url_for("publisher.preservation"))) # create model object to store status details preservation_model = models.PreservationState() preservation_model.set_id() - preservation_model.initiated(current_user.id, f.filename) + previous.insert(0, preservation_model) app.logger.debug(f"Preservation model created with id {preservation_model.id}") if f is None or f.filename == "": - error_str = "No file provided to upload" + error_str = Messages.PRESERVATION_NO_FILE flash(error_str, "error") + preservation_model.initiated(current_user.id, "none") preservation_model.failed(error_str) preservation_model.save() return resp + app.logger.info(f"Preservation file {f.filename}") + + preservation_model.initiated(current_user.id, f.filename) preservation_model.validated() preservation_model.save() diff --git a/portality/view/testdrive.py b/portality/view/testdrive.py index 69ef0d32d4..4fae8b73f2 100644 --- a/portality/view/testdrive.py +++ b/portality/view/testdrive.py @@ -1,4 +1,4 @@ -from flask import Blueprint, make_response, abort, url_for, request +from flask import Blueprint, make_response, abort, url_for, request, render_template from flask_login import current_user, login_required from doajtest.testdrive.factory import TestFactory from portality import util @@ -21,9 +21,13 @@ def testdrive(test_id): params = test.setup() teardown = app.config.get("BASE_URL") + url_for("testdrive.teardown", test_id=test_id) + "?d=" + parse.quote_plus(json.dumps(params)) params["teardown"] = teardown - resp = make_response(json.dumps(params)) - resp.mimetype = "application/json" - return resp + + if request.values.get("json"): + resp = make_response(json.dumps(params)) + resp.mimetype = "application/json" + return resp + + return render_template("testdrive/testdrive.html", params=params, name=test_id) @blueprint.route("//teardown") diff --git a/setup.py b/setup.py index 1c3e15e6ae..b2d6274701 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,12 @@ setup( name='doaj', - version='6.6.2', + version='6.6.10', packages=find_packages(), install_requires=[ "awscli==1.20.50", "bagit==1.8.1", + "beautifulsoup4", "boto3==1.18.50", "elastic-apm==5.2.2", "elasticsearch==7.13.0",