From ff7b16856118de7f50a3af026826dac340d5a408 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 28 Apr 2020 12:41:37 -0300 Subject: [PATCH 1/3] =?UTF-8?q?Aplicativo=20para=20incluir=20DOI=20nos=20r?= =?UTF-8?q?egistros=20h=20e=20f=20de=20uma=20data=20bases=20Cria=20o=20m?= =?UTF-8?q?=C3=B3dulo=20cisis=20e=20testes=20para=20gerar=20comandos=20par?= =?UTF-8?q?a=20atualizar=20uma=20dada=20base=20com=20dados=20de=20DOI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proc/scielo_doi/app/cisis.py | 73 +++++++++++++++++++++++++++ proc/scielo_doi/setup.py | 48 ++++++++++++++++++ proc/scielo_doi/tests/test_cisis.py | 77 +++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 proc/scielo_doi/app/cisis.py create mode 100644 proc/scielo_doi/setup.py create mode 100644 proc/scielo_doi/tests/test_cisis.py diff --git a/proc/scielo_doi/app/cisis.py b/proc/scielo_doi/app/cisis.py new file mode 100644 index 000000000..9833b3d08 --- /dev/null +++ b/proc/scielo_doi/app/cisis.py @@ -0,0 +1,73 @@ +# coding=utf-8 +import os +from datetime import datetime + + +CISIS = "cisis/mx" + + +def update_command(db, mfn, count, proc): + cmd = ("{CISIS} {db} from={mfn} count={count} " + "\"proc={proc}\"" + " copy={db} now -all").format( + CISIS=CISIS, + db=db, + mfn=mfn, + count=count, + proc=proc + ) + return cmd + + +def execute(cmd): + os.system(cmd) + + +def proc_update_doi(file_path, doi, doi_items): + """ + Retorna a "proc" que atualiza + os campos v237 com DOI original e 337 todos os DOI com idioma + """ + if not file_path: + raise ValueError("Invalid value for file_path") + if not doi: + raise ValueError("Invalid value for DOI") + proc_d = "'d337d237a237~{}~'".format(doi) + proc_a337 = [ + ",'a337~^l{}^d{}~'".format(lang, _doi) + for lang, _doi in doi_items + ] + return "if 'hf':v706 and v702='{}' then {} fi".format( + file_path, + proc_d + ''.join(proc_a337) + ) + + +def proc_update_datetime(file_path): + """ + Retorna a "proc" que atualiza + os campos v91 (data) e v92 (hora) da atualização + """ + now = datetime.now().isoformat() + d = now[:10].replace("-", "") + t = now[11:16].replace(":", "") + proc = "'d91d92a91~{}~a92~{}~'".format(d, t) + return "if v706='o' and v702='{}' then {} fi".format( + file_path, + proc + ) + + +def get_commands_to_update_ohf_records_of_a_doc( + db, file_path, h_mfn, doi, doi_items): + """ + Gera dois comandos para atualizar 1 documento os registros: + - h e f: os campos v237 com DOI original e 337 todos os DOI com idioma + - o: os campos v91 (data) e v92 (hora) da atualização + """ + o_mfn = str(int(h_mfn) - 1) + o_proc = proc_update_datetime(file_path) + o_cmd = update_command(db, o_mfn, 1, o_proc) + hf_proc = proc_update_doi(file_path, doi, doi_items) + hf_cmd = update_command(db, h_mfn, 2, hf_proc) + return [o_cmd, hf_cmd] diff --git a/proc/scielo_doi/setup.py b/proc/scielo_doi/setup.py new file mode 100644 index 000000000..7cf91e6a8 --- /dev/null +++ b/proc/scielo_doi/setup.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import os +import setuptools + +here = os.path.abspath(os.path.dirname(__file__)) +README = CHANGES = "" +# with open(os.path.join(here, "README.md")) as f: +# README = f.read() +# with open(os.path.join(here, "CHANGES.txt")) as f: +# CHANGES = f.read() + +requires = [ + +] + +tests_require = [ +] + +setuptools.setup( + name="scielo_doi", + version="0.1", + author="SciELO", + author_email="scielo-dev@googlegroups.com", + description="", + long_description=README + "\n\n" + CHANGES, + long_description_content_type="text/markdown", + license="2-clause BSD", + packages=setuptools.find_packages( + exclude=["*.tests", "*.tests.*", "tests.*", "tests"] + ), + include_package_data=True, + extras_require={"testing": tests_require}, + install_requires=requires, + # dependency_links=[ + # ], + python_requires=">=3.6", + test_suite="tests", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Other Environment", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3 :: Only", + ], + + # entry_points="", +) diff --git a/proc/scielo_doi/tests/test_cisis.py b/proc/scielo_doi/tests/test_cisis.py new file mode 100644 index 000000000..d7c0d27a3 --- /dev/null +++ b/proc/scielo_doi/tests/test_cisis.py @@ -0,0 +1,77 @@ +from unittest import TestCase +from unittest.mock import patch +from datetime import datetime + +from app import cisis + + +class TestCisis(TestCase): + + def test_update_command(self): + db = "base" + mfn = 100 + count = 5 + proc = '@a.proc' + result = cisis.update_command(db, mfn, count, proc) + self.assertEqual( + result, + "cisis/mx base from=100 count=5 \"proc=@a.proc\" copy=base now -all" + ) + + def test_proc_update_doi(self): + file_path = "xml/abc/v1n1/a.xml" + doi = "DOI" + doi_items = [ + ("pt", "doi_PT"), + ("en", "doi-en"), + ("es", "doi_es"), + ] + result = cisis.proc_update_doi(file_path, doi, doi_items) + expected = ("if 'hf':v706 and v702='xml/abc/v1n1/a.xml' then" + " 'd337d237a237~DOI~','a337~^lpt^ddoi_PT~'," + "'a337~^len^ddoi-en~','a337~^les^ddoi_es~' fi") + self.assertEqual(result, expected) + + @patch("app.cisis.datetime") + def test_proc_update_datetime(self, mock_dt): + file_path = "xml/abc/v1n1/a.xml" + mock_dt.now.return_value = datetime(2010, 5, 1, 19, 3, 17) + result = cisis.proc_update_datetime(file_path) + expected = ("if v706='o' and v702='xml/abc/v1n1/a.xml' then" + " 'd91d92a91~20100501~a92~1903~' fi") + self.assertEqual(result, expected) + + @patch("app.cisis.datetime") + def test_get_commands_to_update_ohf_records_of_a_doc(self, mock_dt): + mock_dt.now.return_value = datetime(2010, 5, 1, 19, 3, 17) + db = "base" + file_path = "xml/a.xml" + h_mfn = "4" + doi = "DOI" + doi_items = [ + ("pt", "doi_PT"), + ("en", "doi-en"), + ("es", "doi_es"), + ] + result = cisis.get_commands_to_update_ohf_records_of_a_doc( + db, file_path, h_mfn, doi, doi_items) + + self.assertEqual( + result[0], + ( + "cisis/mx base from=3 count=1 \"proc=" + "if v706='o' and v702='xml/a.xml' then" + " 'd91d92a91~20100501~a92~1903~' fi\" " + "copy=base now -all" + ) + ) + self.assertEqual( + result[1], + ( + "cisis/mx base from=4 count=2 \"proc=" + "if 'hf':v706 and v702='xml/a.xml' then" + " 'd337d237a237~DOI~','a337~^lpt^ddoi_PT~'," + "'a337~^len^ddoi-en~','a337~^les^ddoi_es~' fi\" " + "copy=base now -all" + ) + ) From 09c88dc1f013166f6f06cedfe584c532aaa384da Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 5 May 2020 17:03:17 -0300 Subject: [PATCH 2/3] =?UTF-8?q?Melhorias=20no=20c=C3=B3digo=20e=20nos=20te?= =?UTF-8?q?stes=20Remove=20a=20restricao=20de=20versao=20do=20python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proc/scielo_doi/app/cisis.py | 14 ++++----- proc/scielo_doi/setup.py | 5 --- proc/scielo_doi/tests/test_cisis.py | 49 +++++++++++++++-------------- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/proc/scielo_doi/app/cisis.py b/proc/scielo_doi/app/cisis.py index 9833b3d08..dbdad82e0 100644 --- a/proc/scielo_doi/app/cisis.py +++ b/proc/scielo_doi/app/cisis.py @@ -1,6 +1,5 @@ # coding=utf-8 import os -from datetime import datetime CISIS = "cisis/mx" @@ -20,6 +19,8 @@ def update_command(db, mfn, count, proc): def execute(cmd): + # TODO usar subprocess + # TODO remover esta funcao deste modulo os.system(cmd) @@ -43,15 +44,12 @@ def proc_update_doi(file_path, doi, doi_items): ) -def proc_update_datetime(file_path): +def proc_update_datetime(file_path, update_date, update_time): """ Retorna a "proc" que atualiza os campos v91 (data) e v92 (hora) da atualização """ - now = datetime.now().isoformat() - d = now[:10].replace("-", "") - t = now[11:16].replace(":", "") - proc = "'d91d92a91~{}~a92~{}~'".format(d, t) + proc = "'d91d92a91~{}~a92~{}~'".format(update_date, update_time) return "if v706='o' and v702='{}' then {} fi".format( file_path, proc @@ -59,14 +57,14 @@ def proc_update_datetime(file_path): def get_commands_to_update_ohf_records_of_a_doc( - db, file_path, h_mfn, doi, doi_items): + db, file_path, h_mfn, doi, doi_items, update_date, update_time): """ Gera dois comandos para atualizar 1 documento os registros: - h e f: os campos v237 com DOI original e 337 todos os DOI com idioma - o: os campos v91 (data) e v92 (hora) da atualização """ o_mfn = str(int(h_mfn) - 1) - o_proc = proc_update_datetime(file_path) + o_proc = proc_update_datetime(file_path, update_date, update_time) o_cmd = update_command(db, o_mfn, 1, o_proc) hf_proc = proc_update_doi(file_path, doi, doi_items) hf_cmd = update_command(db, h_mfn, 2, hf_proc) diff --git a/proc/scielo_doi/setup.py b/proc/scielo_doi/setup.py index 7cf91e6a8..1329264c2 100644 --- a/proc/scielo_doi/setup.py +++ b/proc/scielo_doi/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import os import setuptools @@ -33,15 +32,11 @@ install_requires=requires, # dependency_links=[ # ], - python_requires=">=3.6", test_suite="tests", classifiers=[ "Development Status :: 2 - Pre-Alpha", "Environment :: Other Environment", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3 :: Only", ], # entry_points="", diff --git a/proc/scielo_doi/tests/test_cisis.py b/proc/scielo_doi/tests/test_cisis.py index d7c0d27a3..1ee4660f7 100644 --- a/proc/scielo_doi/tests/test_cisis.py +++ b/proc/scielo_doi/tests/test_cisis.py @@ -1,6 +1,5 @@ from unittest import TestCase -from unittest.mock import patch -from datetime import datetime + from app import cisis @@ -20,41 +19,41 @@ def test_update_command(self): def test_proc_update_doi(self): file_path = "xml/abc/v1n1/a.xml" - doi = "DOI" + doi = "10.1590/1809-4392201902421" doi_items = [ - ("pt", "doi_PT"), - ("en", "doi-en"), - ("es", "doi_es"), + ("pt", "10.1590/1809-4392201902421.1"), + ("en", "10.1590/1809-4392201902421.2"), + ("es", "10.1590/1809-4392201902421.3"), ] result = cisis.proc_update_doi(file_path, doi, doi_items) - expected = ("if 'hf':v706 and v702='xml/abc/v1n1/a.xml' then" - " 'd337d237a237~DOI~','a337~^lpt^ddoi_PT~'," - "'a337~^len^ddoi-en~','a337~^les^ddoi_es~' fi") + expected = ("if 'hf':v706 and v702='xml/abc/v1n1/a.xml' then " + "'d337d237a237~10.1590/1809-4392201902421.1~'," + "'a337~^lpt^d10.1590/1809-4392201902421.1~'," + "'a337~^len^d10.1590/1809-4392201902421.2~'," + "'a337~^les^d10.1590/1809-4392201902421.3~' fi") self.assertEqual(result, expected) - @patch("app.cisis.datetime") - def test_proc_update_datetime(self, mock_dt): + def test_proc_update_datetime(self): file_path = "xml/abc/v1n1/a.xml" - mock_dt.now.return_value = datetime(2010, 5, 1, 19, 3, 17) - result = cisis.proc_update_datetime(file_path) + result = cisis.proc_update_datetime(file_path, '20100501', '1903') expected = ("if v706='o' and v702='xml/abc/v1n1/a.xml' then" " 'd91d92a91~20100501~a92~1903~' fi") self.assertEqual(result, expected) - @patch("app.cisis.datetime") - def test_get_commands_to_update_ohf_records_of_a_doc(self, mock_dt): - mock_dt.now.return_value = datetime(2010, 5, 1, 19, 3, 17) + def test_get_commands_to_update_ohf_records_of_a_doc(self): db = "base" file_path = "xml/a.xml" h_mfn = "4" - doi = "DOI" + doi = "10.1590/1809-4392201902421.1" doi_items = [ - ("pt", "doi_PT"), - ("en", "doi-en"), - ("es", "doi_es"), + ("pt", "10.1590/1809-4392201902421.1"), + ("en", "10.1590/1809-4392201902421.2"), + ("es", "10.1590/1809-4392201902421.3"), ] + upd_date = '20100501' + upd_time = '1903' result = cisis.get_commands_to_update_ohf_records_of_a_doc( - db, file_path, h_mfn, doi, doi_items) + db, file_path, h_mfn, doi, doi_items, upd_date, upd_time) self.assertEqual( result[0], @@ -69,9 +68,11 @@ def test_get_commands_to_update_ohf_records_of_a_doc(self, mock_dt): result[1], ( "cisis/mx base from=4 count=2 \"proc=" - "if 'hf':v706 and v702='xml/a.xml' then" - " 'd337d237a237~DOI~','a337~^lpt^ddoi_PT~'," - "'a337~^len^ddoi-en~','a337~^les^ddoi_es~' fi\" " + "if 'hf':v706 and v702='xml/a.xml' then " + "'d337d237a237~10.1590/1809-4392201902421.1~'," + "'a337~^lpt^d10.1590/1809-4392201902421.1~'," + "'a337~^len^d10.1590/1809-4392201902421.2~'," + "'a337~^les^d10.1590/1809-4392201902421.3~' fi\" " "copy=base now -all" ) ) From 6e964ff3e13aa86356bf26c4333a0c586382004c Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Wed, 6 May 2020 10:13:22 -0300 Subject: [PATCH 3/3] Correcoes solicitadas que faltaram --- proc/scielo_doi/setup.py | 27 --------------------------- proc/scielo_doi/tests/__init__.py | 0 proc/scielo_doi/tests/test_cisis.py | 2 +- 3 files changed, 1 insertion(+), 28 deletions(-) create mode 100644 proc/scielo_doi/tests/__init__.py diff --git a/proc/scielo_doi/setup.py b/proc/scielo_doi/setup.py index 1329264c2..a123b91a2 100644 --- a/proc/scielo_doi/setup.py +++ b/proc/scielo_doi/setup.py @@ -1,19 +1,5 @@ -import os import setuptools -here = os.path.abspath(os.path.dirname(__file__)) -README = CHANGES = "" -# with open(os.path.join(here, "README.md")) as f: -# README = f.read() -# with open(os.path.join(here, "CHANGES.txt")) as f: -# CHANGES = f.read() - -requires = [ - -] - -tests_require = [ -] setuptools.setup( name="scielo_doi", @@ -21,23 +7,10 @@ author="SciELO", author_email="scielo-dev@googlegroups.com", description="", - long_description=README + "\n\n" + CHANGES, - long_description_content_type="text/markdown", license="2-clause BSD", packages=setuptools.find_packages( exclude=["*.tests", "*.tests.*", "tests.*", "tests"] ), include_package_data=True, - extras_require={"testing": tests_require}, - install_requires=requires, - # dependency_links=[ - # ], test_suite="tests", - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Environment :: Other Environment", - "License :: OSI Approved :: BSD License", - ], - - # entry_points="", ) diff --git a/proc/scielo_doi/tests/__init__.py b/proc/scielo_doi/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/proc/scielo_doi/tests/test_cisis.py b/proc/scielo_doi/tests/test_cisis.py index 1ee4660f7..20b5d5aae 100644 --- a/proc/scielo_doi/tests/test_cisis.py +++ b/proc/scielo_doi/tests/test_cisis.py @@ -27,7 +27,7 @@ def test_proc_update_doi(self): ] result = cisis.proc_update_doi(file_path, doi, doi_items) expected = ("if 'hf':v706 and v702='xml/abc/v1n1/a.xml' then " - "'d337d237a237~10.1590/1809-4392201902421.1~'," + "'d337d237a237~10.1590/1809-4392201902421~'," "'a337~^lpt^d10.1590/1809-4392201902421.1~'," "'a337~^len^d10.1590/1809-4392201902421.2~'," "'a337~^les^d10.1590/1809-4392201902421.3~' fi")