add the function of using html class tag to extract infobox

requirements.txt

@@ -1,3 +1,4 @@
 certifi>=2017.7.27.1
 html2text>=2016.9.19
 lxml>=3.8.0
+beautifulsoup4

tests/test_advanced.py

@@ -99,6 +99,12 @@ def test_not_found(self):
         except LookupError as detail:
             pass
 
+    def test_finding_infobox_with_tag(self) :
+        page = wptools.page("秋田県民会館", lang='ja', boxtag='infobox')
+        page.get_parse(show=False)
+        infobox = page.data['infobox']
+        self.assertTrue(infobox is not None)
+
     def test_lookup_unicode_error(self):
         """
         Raise LookupError without UnicodeDecodeError. Issue #29

tests/test_basic.py

@@ -424,6 +424,7 @@ def test_page_get_parse_66(self):
         self.assertEqual(len(infobox['Genre'].split('<br')), 8)
         self.assertTrue('requests' not in page.data)
 
+
     def test_page_get_parse_91(self):
         """
         Get infobox data with unusual wikitext syntax

wptools/page.py

@@ -64,6 +64,10 @@ def __init__(self, *args, **kwargs):
         if boxterm:
             self.params.update({'boxterm': boxterm})
 
+        boxtag = kwargs.get('boxtag')
+        if boxtag:
+            self.params.update({'boxtag':boxtag})
+
         endpoint = kwargs.get('endpoint')
         if endpoint:
             self.params.update({'endpoint': endpoint})
@@ -250,11 +254,15 @@ def _set_parse_data(self):
         parsetree = pdata.get('parsetree')
         self.data['parsetree'] = parsetree
 
-        boxterm = self.params.get('boxterm')
-        if boxterm:
-            infobox = utils.get_infobox(parsetree, boxterm)
+        boxtag = self.params.get('boxtag')
+        if boxtag:
+            infobox = utils.get_infoboxes_withtag(pdata['text'], parsetree, boxtag)
         else:
-            infobox = utils.get_infobox(parsetree)
+            boxterm = self.params.get('boxterm')
+            if boxterm:
+                infobox = utils.get_infobox(parsetree, boxterm)
+            else:
+                infobox = utils.get_infobox(parsetree)
         self.data['infobox'] = infobox
 
         title = pdata.get('title')

wptools/utils.py

@@ -18,6 +18,58 @@
 
 from lxml.etree import tostring
 
+from bs4 import BeautifulSoup
+
+
+def get_boxterm_with_boxtag(pagetxt,ptree,boxtag='infobox'):
+    soup = BeautifulSoup(pagetxt)
+    # here not ensure all infobox items are tables
+    ptree_soup = BeautifulSoup(ptree)
+
+    table_first_string = soup.find('table', class_=boxtag).find('th').text.strip()
+
+
+
+def order_keep_same(text1, text2) :
+    for i in text1 :
+        try :
+            text2 = text2[text2.index(i) + 1 :]
+        except :
+            return False
+    return True
+
+def get_infoboxes_withtag(pagetxt, ptree, boxtag='infobox', multi=True) :
+    soup = BeautifulSoup(pagetxt)
+    # here not ensure all infobox items are tables, still some items cannot be acquires.
+    ptree_soup = BeautifulSoup(ptree)
+
+    table_first_string = soup.find('table', class_=boxtag).find('th').text.strip()
+
+    boxes = []
+
+    # this function will get all infoboxes, and match them to all templates, distinguish with box titles;
+    for temp_item in ptree_soup.findAll('template') :
+        try:
+            if order_keep_same(table_first_string, temp_item.text.strip()) :
+
+                title = temp_item.find('title').text
+                item = lxml.etree.fromstring(str(temp_item))
+                box = template_to_dict(item)
+
+                if box :
+                    if not multi:
+                        return box
+
+                alt = template_to_dict_alt(item, title)
+                if alt :
+                    boxes.append(alt)
+
+        except:
+            pass
+
+    if boxes :
+        return {'boxes' : boxes, 'count' : len(boxes)}
+
 
 def get_infobox(ptree, boxterm="box"):
     """