Converting XML to JSON

The index API that Elasticsearch uses adds or updates a typed JSON document in a specific index, making it searchable. So our first objectives was converting each article in the XML files to a JSON file. To do this we use xml.etree.ElementTree. This makes it so we can iterate over each article, while creating a new article in JSON format (and optionally also create a word cloud). All these new articles in JSON format are appended together and saved as a large JSON file, which is later used to bulk index!

def xml_to_json(wordcloud):
    for infile in glob.glob(os.path.join("XML", "*.xml")):
        print infile
        all_articles = []
        filename = os.path.splitext(os.path.basename(infile))[0]
        tree = ET.iterparse(infile)
        for _, element in tree:
            if element.tag == "{http://www.politicalmashup.nl}root":
                new_article = {
                    "date": element[1][0].text,
                    "subject": element[1][1].text,
                    "title": element[2][0].text,
                    "text": " ".join([child.text for child in element[2][1] if child.text is not None]),
                    "id": element[2].attrib["{http://www.politicalmashup.nl}id"],
                    "source": element[2].attrib["{http://www.politicalmashup.nl}source"]
                }
                if wordcloud:
                    create_wordcloud.create_cloud(new_article["text"], "".join(new_article["id"].split(":")))
                all_articles.append(new_article)
                element.clear()

        with open("JSON/{}.json".format(filename), "w") as f:
                    json.dump(all_articles, f, indent=4)