Skip to content
This repository
Browse code

added -t type option, removed -f subfield option

  • Loading branch information...
commit 9f20b191fc0c6a3987623a7f0bde0b0a338ed20f 1 parent ea51868
Luciano Ramalho ramalho authored
BIN  fixtures/lilacs1/LILACS.cnt
Binary file not shown
28 fixtures/lilacs1/LILACS.fdt
... ... @@ -0,0 +1,28 @@
  1 +W:LILACS
  2 +F:LILACS
  3 +S:LILACS
  4 +***
  5 +one 1 1000 0 0
  6 +two 2 1000 2 0
  7 +four 4 1000 0 1
  8 +five 5 1000 0 0
  9 +six 6 1000 0 0
  10 +eight 8 1000 0 0
  11 +ten 10 1000 0 1
  12 +twelve 12 1000 0 1
  13 +thirty 30 1000 0 0
  14 +thirty-one 31 1000 0 0
  15 +thirty-two 32 1000 0 0
  16 +thirty-five 35 1000 0 0
  17 +forty 40 1000 0 0
  18 +forty-one 41 1000 0 1
  19 +sixty-four 64 1000 0 0
  20 +sixty-five 65 1000 0 0
  21 +sixty-seven 67 1000 0 0
  22 +eighty-three 83 1000 0 1
  23 +eighty-four 84 1000 0 0
  24 +ninety-one 91 1000 0 0
  25 +ninety-two 92 1000 0 1
  26 +ninety-eight 98 1000 0 0
  27 +one-thirteen 113 1000 0 0
  28 +seven-seven-eight 778 1000 0 0
BIN  fixtures/lilacs1/LILACS.ifp
Binary file not shown
35 fixtures/lilacs1/LILACS.iso
... ... @@ -0,0 +1,35 @@
  1 +02727000000000409000450000100060000000200070000600400070001300400080002000500020
  2 +00280060003000300080107000330100162001400100145003020100149004470100147005960120
  3 +06000743012006800803030002100871031000200892032000200894035001000896040000300906
  4 +04100030090904100030091206400110091506500090092606700030093508306510093808306350
  5 +15890840011022240910009022350920009022440920004022530980006022571130002022637780
  6 +05202265BR1.1538886LILACSLLXPEDTSasInternet^ihttp://www.demneuropsy.com.b
  7 +r/imageBank/PDF/v3n3a04.pdf?aid2=168&nome_en=v3n3a04.pdf^len.pdfKanda, Paul
  8 +o Afonso de Medeiros^1University of Sao Paulo ^2School of Medicine^3Cognitive Di
  9 +sorders of Clinicas Hospital Reference Center^pBrasil ^cSao Paulo^rorgAnghinah,
  10 + Renato^1University of Sao Paulo^2School of Medicine^3Cognitive Disorders of Cli
  11 +nicas Hospital Reference Center^pBrasil^cSao Paulo^rorgSmidth, Magali Taino^1Un
  12 +iversity of Sao Paulo^2School of Medicine^3Cognitive Disorders of Clinicas Hospi
  13 +tal Reference Center^pBrasil^cSao Paulo^rorgSilva, Jorge Mario^1University of S
  14 +ao Paulo^2School of Medicine^3Cognitive Disorders of Clinicas Hospital Reference
  15 + Center^pBrasil^cSao Paulo^rorgThe Clinical use of quantitative EEG in cognitiv
  16 +e disordersA utiliza�ao cl�nica do EEG quantitativo nos transtornos cognitivos
  17 +Dement. neuropsychol331980-5764PtEnEn20/09/200920090000BRAbstract: The
  18 + primary diagnosis of most cognitive disorders is clinically based, but the EEG
  19 +plays a role in evaluating, classifying and following some of these disorders. T
  20 +here is an ongoing debate over routine use of qEEG. Although many findings regar
  21 +ding the clinical use of quantitative EEG are awaiting validation by independent
  22 + investigators while confirmatory clinical follow-up studies are also needed, qE
  23 +EG can be cautiously used by a skilled neurophysiologist in cognitive dysfunctio
  24 +ns to improve the analysis of background activity, slow/fast focal activity, sub
  25 +tle asymmetries, spikes and waves, as well as in longitudinal follow-ups.(AU)Re
  26 +sumo: O uso cl�nico do EEG Quantitativo nas doen�as cognitivas. O diagn�stico da
  27 +s doen�as cognitivas geralmente � cl�nico mas o EEG � importante como exame auxi
  28 +liar na avalia�ao, diagn�stico e classifica�ao de algumas delas. O debate atual
  29 +refere-se ao uso cl�nico do EEGq. Embora muitos achados no EEGq ainda aguardem v
  30 +alida�ao, o EEGq pode ser usado cautelosamente em situa�oes espec�ficas e por um
  31 + neurofisiologista experiente. Nas doen�as cognitivas ele pode contribuir na an�
  32 +lise da atividade de base, em atividades focais lentas ou r�pidas, assimetrias s
  33 +ut�s, pontas e ondas e no acompanhamento longitudinal dos pacientes.(AU)2010-03
  34 +-0520100211BR1876.9EDTFONTEp538886^dBIREME_LLXPEDT^sS1980-5764200900040001
  35 +00015
BIN  fixtures/lilacs1/LILACS.mst
Binary file not shown
25 fixtures/lilacs1/LILACS.pft
... ... @@ -0,0 +1,25 @@
  1 +f2,cl4,m(3175,-3175)
  2 +"one (1):",tab(3175),v1/
  3 +"two (2):",tab(3175),v2/
  4 +(if p(v4) then 'four (4):',tab(3175),v4/fi)
  5 +"five (5):",tab(3175),v5/
  6 +"six (6):",tab(3175),v6/
  7 +"eight (8):",tab(3175),v8/
  8 +(if p(v10) then 'ten (10):',tab(3175),v10/fi)
  9 +(if p(v12) then 'twelve (12):',tab(3175),v12/fi)
  10 +"thirty (30):",tab(3175),v30/
  11 +"thirty-one (31):",tab(3175),v31/
  12 +"thirty-two (32):",tab(3175),v32/
  13 +"thirty-five (35):",tab(3175),v35/
  14 +"forty (40):",tab(3175),v40/
  15 +(if p(v41) then 'forty-one (41):',tab(3175),v41/fi)
  16 +"sixty-four (64):",tab(3175),v64/
  17 +"sixty-five (65):",tab(3175),v65/
  18 +"sixty-seven (67):",tab(3175),v67/
  19 +(if p(v83) then 'eighty-three (83):',tab(3175),v83/fi)
  20 +"eighty-four (84):",tab(3175),v84/
  21 +"ninety-one (91):",tab(3175),v91/
  22 +(if p(v92) then 'ninety-two (92):',tab(3175),v92/fi)
  23 +"ninety-eight (98):",tab(3175),v98/
  24 +"one-thirteen (113):",tab(3175),v113/
  25 +"seven-seven-eight (778):",tab(3175),v778/
BIN  fixtures/lilacs1/LILACS.xrf
Binary file not shown
47 tools/isis2json.py
@@ -41,7 +41,7 @@
41 41 SUBFIELD_DELIMITER = '^'
42 42 INPUT_ENCODING = 'cp1252'
43 43
44   -def iterMstRecords(master_file_name, subfields):
  44 +def iterMstRecords(master_file_name, isis_json_type):
45 45 try:
46 46 from br.bireme.zeus.master import MasterFactory, Record
47 47 except ImportError:
@@ -59,7 +59,7 @@ def iterMstRecords(master_file_name, subfields):
59 59 for field in record.getFields():
60 60 field_key = str(field.getId())
61 61 field_occurrences = fields.setdefault(field_key,[])
62   - if subfields:
  62 + if isis_json_type == 3:
63 63 content = {}
64 64 for subfield in field.getSubfields():
65 65 subfield_key = subfield.getId()
@@ -69,7 +69,7 @@ def iterMstRecords(master_file_name, subfields):
69 69 subfield_occurrences = content.setdefault(subfield_key,[])
70 70 subfield_occurrences.append(subfield.getContent())
71 71 field_occurrences.append(content)
72   - else:
  72 + elif isis_json_type == 1:
73 73 content = []
74 74 for subfield in field.getSubfields():
75 75 subfield_key = subfield.getId()
@@ -79,22 +79,14 @@ def iterMstRecords(master_file_name, subfields):
79 79 content.append(SUBFIELD_DELIMITER+subfield_key+
80 80 subfield.getContent())
81 81 field_occurrences.append(''.join(content))
  82 + else:
  83 + raise NotImplementedError('ISIS-JSON type %s conversion not yet implemented for .mst input' % isis_json_type)
82 84 yield fields
83 85 mst.close()
84 86
85   -def iterIsoRecords(iso_file_name, subfields):
  87 +def iterIsoRecords(iso_file_name, isis_json_type):
86 88 from iso2709 import IsoFile
87   - def parse(field):
88   - content = field.value.decode(INPUT_ENCODING,'replace')
89   - parts = content.split(SUBFIELD_DELIMITER)
90   - subs = {}
91   - main = parts.pop(0)
92   - if len(main) > 0:
93   - subs['_'] = main
94   - for part in parts:
95   - prefix = part[0]
96   - subs[prefix] = part[1:]
97   - return subs
  89 + from subfield import expand
98 90
99 91 iso = IsoFile(iso_file_name)
100 92 for record in iso:
@@ -102,16 +94,21 @@ def parse(field):
102 94 for field in record.directory:
103 95 field_key = str(int(field.tag)) # remove leading zeroes
104 96 field_occurrences = fields.setdefault(field_key,[])
105   - if subfields:
106   - field_occurrences.append(parse(field))
  97 + content = field.value.decode(INPUT_ENCODING,'replace')
  98 + if isis_json_type == 1:
  99 + field_occurrences.append(content)
  100 + elif isis_json_type == 2:
  101 + field_occurrences.append(expand(content))
  102 + elif isis_json_type == 3:
  103 + field_occurrences.append(dict(expand(content)))
107 104 else:
108   - field_occurrences.append(field.value.decode(INPUT_ENCODING,'replace'))
  105 + raise NotImplementedError('ISIS-JSON type %s conversion not yet implemented for .iso input' % isis_json_type)
109 106
110 107 yield fields
111 108 iso.close()
112 109
113 110 def writeJsonArray(iterRecords, file_name, output, qty, skip, id_tag,
114   - gen_uuid, mongo, mfn, subfields, prefix, constant):
  111 + gen_uuid, mongo, mfn, isis_json_type, prefix, constant):
115 112 start = skip
116 113 end = start + qty
117 114 if not mongo:
@@ -121,7 +118,7 @@ def writeJsonArray(iterRecords, file_name, output, qty, skip, id_tag,
121 118 ids = set()
122 119 else:
123 120 id_tag = ''
124   - for i, record in enumerate(iterRecords(file_name, subfields)):
  121 + for i, record in enumerate(iterRecords(file_name, isis_json_type)):
125 122 if i >= end:
126 123 break
127 124 if i > start and not mongo:
@@ -192,12 +189,8 @@ def writeJsonArray(iterRecords, file_name, output, qty, skip, id_tag,
192 189 help='output individual records as separate JSON dictionaries,'
193 190 ' one per line for bulk insert to MongoDB via mongoimport utility')
194 191 parser.add_argument(
195   - '-f', '--subfields', action='store_true',
196   - help='explode each field into a JSON dictionary, with "_" as'
197   - ' default key, and subfield markers as additional keys')
198   - #parser.add_argument(
199   - # '-t', '--type', type=int, metavar='ISIS_JSON_TYPE', default=1,
200   - # help='ISIS-JSON type, sets field structure: 1=string, 2=alist, 3=dict')
  192 + '-t', '--type', type=int, metavar='ISIS_JSON_TYPE', default=1,
  193 + help='ISIS-JSON type, sets field structure: 1=string, 2=alist, 3=dict')
201 194 parser.add_argument(
202 195 '-q', '--qty', type=int, default=DEFAULT_QTY,
203 196 help='maximum quantity of records to read (default=ALL)')
@@ -241,7 +234,7 @@ def writeJsonArray(iterRecords, file_name, output, qty, skip, id_tag,
241 234 if args.couch:
242 235 args.out.write('{ "docs" : ')
243 236 writeJsonArray(iterRecords, args.file_name, args.out, args.qty, args.skip,
244   - args.id, args.uuid, args.mongo, args.mfn, args.subfields, args.prefix, args.constant)
  237 + args.id, args.uuid, args.mongo, args.mfn, args.type, args.prefix, args.constant)
245 238 if args.couch:
246 239 args.out.write('}\n')
247 240 args.out.close()
7 tools/test.sh
... ... @@ -0,0 +1,7 @@
  1 +#!/bin/bash
  2 +
  3 +PYTHONPATH=../isisdm/:$PYTHONPATH
  4 +
  5 +python isis2json.py -t $1 ../fixtures/lilacs1/LILACS.iso
  6 +
  7 +#jython isis2json.py -t $1 ../fixtures/lilacs1/LILACS.mst

0 comments on commit 9f20b19

Please sign in to comment.
Something went wrong with that request. Please try again.