IndexError: list index out of range

[tsoernes]

Describe the bug

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/core.py:443, in Table.set_edges(self, vertical, horizontal, joint_tol)
    441     i = len(self.rows) - 1
    442     for j in range(start, end):
--> 443         self.cells[i][j].bottom = True
    444 elif i == 0:  # only top edge
    445     for j in range(start, end):

IndexError: list index out of range

Steps to reproduce the bug

Expected behavior

Code

    tables_li = pypdf_table_extraction.read_pdf(
        str(pdf_path),
        pages=page_numbers,
        strip_text="\n",
    )

PDF

Cannot add, confidential

Screenshots

Environment

• OS: [e.g. macOS]
• Python version: 3.12
• Numpy version: '1.26.4'
• OpenCV version:
• Ghostscript version: GPL Ghostscript 10.02.1 (2023-11-01)
• pypdf_table_extraction version: 0.0.2

Additional context

Full traceback

2024-10-21T09:47:14 - INFO - Processing page-149
Processing page-149
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
File ~/code/ai-pilot-wellreport/table_extraction/no_conda/extraction.py:175
    171         page_to_tables[table.page].append(TableInfo(table._bbox[1], table._bbox[3], df))
    172     return dict(page_to_tables)
--> 175 page_to_tables = get_tables(pdf_path, page_numbers)
    176 markdown_content = pdf_to_markdown(pdf_path, page_numbers, page_to_tables)
    177 print(markdown_content)

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/extraction.py:163, in get_tables(pdf_path, page_numbers)
    161 elif isinstance(page_numbers, list):
    162     page_numbers = ",".join(map(str, page_numbers))
--> 163 tables_li = camelot.read_pdf(
    164     str(pdf_path),
    165     pages=page_numbers,
    166     strip_text="\n",
    167 )
    168 page_to_tables = defaultdict(list)
    169 for table in tables_li:

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/io.py:125, in read_pdf(filepath, pages, password, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    123 p = PDFHandler(filepath, pages=pages, password=password)
    124 kwargs = remove_extra(kwargs, flavor=flavor)
--> 125 tables = p.parse(
    126     flavor=flavor,
    127     suppress_stdout=suppress_stdout,
    128     parallel=parallel,
    129     layout_kwargs=layout_kwargs,
    130     **kwargs
    131 )
    132 return tables

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/handlers.py:202, in PDFHandler.parse(self, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    200     else:
    201         for p in self.pages:
--> 202             t = self._parse_page(
    203                 p, tempdir, parser, suppress_stdout, layout_kwargs
    204             )
    205             tables.extend(t)
    207 return TableList(sorted(tables))

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/handlers.py:232, in PDFHandler._parse_page(self, page, tempdir, parser, suppress_stdout, layout_kwargs)
    230 self._save_page(self.filepath, page, tempdir)
    231 page_path = os.path.join(tempdir, f"page-{page}.pdf")
--> 232 tables = parser.extract_tables(
    233     page_path, suppress_stdout=suppress_stdout, layout_kwargs=layout_kwargs
    234 )
    235 return tables

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/parsers/lattice.py:412, in Lattice.extract_tables(self, filename, suppress_stdout, layout_kwargs)
    408 for table_idx, tk in enumerate(
    409     sorted(self.table_bbox.keys(), key=lambda x: x[1], reverse=True)
    410 ):
    411     cols, rows, v_s, h_s = self._generate_columns_and_rows(table_idx, tk)
--> 412     table = self._generate_table(table_idx, cols, rows, v_s=v_s, h_s=h_s)
    413     table._bbox = tk
    414     _tables.append(table)

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/parsers/lattice.py:334, in Lattice._generate_table(self, table_idx, cols, rows, **kwargs)
    332 table = Table(cols, rows)
    333 # set table edges to True using ver+hor lines
--> 334 table = table.set_edges(v_s, h_s, joint_tol=self.joint_tol)
    335 # set table border edges to True
    336 table = table.set_border()

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/core.py:443, in Table.set_edges(self, vertical, horizontal, joint_tol)
    441     i = len(self.rows) - 1
    442     for j in range(start, end):
--> 443         self.cells[i][j].bottom = True
    444 elif i == 0:  # only top edge
    445     for j in range(start, end):

IndexError: list index out of range

[bosd]

Hmm, is this parser related?
Can you try the same file with the network parser?

[tsoernes]

@bosd How do I do that?

[bosd]

By adding flavor="network", see api reference

    tables_li = pypdf_table_extraction.read_pdf(
        flavor="network",
        str(pdf_path),
        pages=page_numbers,
        strip_text="\n",
    )

please also try the variant without strip_text="\n",. So we can pinpoint more precisely why the error occurs.
It's kinda hard to debug with the PDF input file.
Is it possible to replicate the input file structure with anonymized/bogus data?

[tsoernes]

The network extractor work. (edit: network parser)

The regular extractor (edit: i.e. lattice parser) still does not work, but I get a different error on the newest commit (as of now, 0.0.2 88a3c3d)

In [19]: tables_li = pypdf_table_extraction.read_pdf(flavor="network", filepath=str(pdf_path), pages="149", strip_text="\n")

In [20]: tables_li = pypdf_table_extraction.read_pdf(filepath=str(pdf_path), pages="149", strip_text="\n")
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[20], line 1
----> 1 tables_li = pypdf_table_extraction.read_pdf(filepath=str(pdf_path), pages="149", strip_text="\n")

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/io.py:134, in read_pdf(filepath, pages, password, flavor, suppress_stdout, parallel, layout_kwargs, debug, **kwargs)
    132 p = PDFHandler(filepath, pages=pages, password=password, debug=debug)
    133 kwargs = remove_extra(kwargs, flavor=flavor)
--> 134 tables = p.parse(
    135     flavor=flavor,
    136     suppress_stdout=suppress_stdout,
    137     parallel=parallel,
    138     layout_kwargs=layout_kwargs,
    139     **kwargs
    140 )
    141 return tables

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py:237, in PDFHandler.parse(self, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    235     else:
    236         for p in self.pages:
--> 237             t = self._parse_page(
    238                 p, tempdir, parser, suppress_stdout, layout_kwargs
    239             )
    240             tables.extend(t)
    242 return TableList(sorted(tables))

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py:270, in PDFHandler._parse_page(self, page, tempdir, parser, suppress_stdout, layout_kwargs)
    266 layout, dimensions = get_page_layout(page_path, **layout_kwargs)
    267 parser.prepare_page_parse(
    268     page_path, layout, dimensions, page, layout_kwargs=layout_kwargs
    269 )
--> 270 tables = parser.extract_tables()
    271 return tables

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py:229, in BaseParser.extract_tables(self)
    226         user_cols = None
    228     cols, rows, v_s, h_s = self._generate_columns_and_rows(bbox, user_cols)
--> 229     table = self._generate_table(table_idx, bbox, cols, rows, v_s=v_s, h_s=h_s)
    230     _tables.append(table)
    232 return _tables

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/lattice.py:365, in Lattice._generate_table(self, table_idx, bbox, cols, rows, **kwargs)
    363 table = table.set_edges(v_s, h_s, joint_tol=self.joint_tol)
    364 # set table border edges to True
--> 365 table = table.set_border()
    367 self.record_parse_metadata(table)
    368 return table

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/core.py:733, in Table.set_border(self)
    731     self.cells[index][len(self.cols) - 1].right = True
    732 for index, _col in enumerate(self.cols):
--> 733     self.cells[0][index].top = True
    734     self.cells[len(self.rows) - 1][index].bottom = True
    735 return self

IndexError: list index out of range

In [21]: 

[bosd]

What do you mean with regular extractor?

[tsoernes]

Sorry, I meant the default "lattice" parser.

By the way, the network parser also fails on my PDF, but on a different page. It succeeds on the page that the lattice parser failed on.

---------------------------------------------------------------------------
RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/home/torstein/.pixi/envs/pipx/lib/python3.12/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py", line 270, in _parse_page
    tables = parser.extract_tables()
             ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 229, in extract_tables
    table = self._generate_table(table_idx, bbox, cols, rows, v_s=v_s, h_s=h_s)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 470, in _generate_table
    self.record_parse_metadata(table)
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 476, in record_parse_metadata
    super().record_parse_metadata(table)
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 240, in record_parse_metadata
    pos_errors = self.compute_parse_errors(table)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 185, in compute_parse_errors
    if indices[0][:2] != (-1, -1):
       ~~~~~~~^^^
IndexError: list index out of range
"""

The above exception was the direct cause of the following exception:

IndexError                                Traceback (most recent call last)

---> 54 tables = pypdf_table_extraction.read_pdf(  # type: ignore
     55     str(pdf.path),
     56     flavor="network",
     57     pages=pdf.page_numbers_str,
     58     strip_text="\n",
     59     parallel=True,
     60 )
     61 for table in tables:
     62     df = prep_dataframe(table.df)

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/io.py:134, in read_pdf(filepath, pages, password, flavor, suppress_stdout, parallel, layout_kwargs, debug, **kwargs)
    132 p = PDFHandler(filepath, pages=pages, password=password, debug=debug)
    133 kwargs = remove_extra(kwargs, flavor=flavor)
--> 134 tables = p.parse(
    135     flavor=flavor,
    136     suppress_stdout=suppress_stdout,
    137     parallel=parallel,
    138     layout_kwargs=layout_kwargs,
    139     **kwargs
    140 )
    141 return tables

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py:233, in PDFHandler.parse(self, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    230             jobs.append(j)
    232         for j in jobs:
--> 233             t = j.get()
    234             tables.extend(t)
    235 else:

File ~/.pixi/envs/pipx/lib/python3.12/multiprocessing/pool.py:774, in ApplyResult.get(self, timeout)
    772     return self._value
    773 else:
--> 774     raise self._value

IndexError: list index out of range

[tsoernes]

Stream parser fails with:

RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/home/torstein/.pixi/envs/pipx/lib/python3.12/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py", line 270, in _parse_page
    tables = parser.extract_tables()
             ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 229, in extract_tables
    table = self._generate_table(table_idx, bbox, cols, rows, v_s=v_s, h_s=h_s)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 470, in _generate_table
    self.record_parse_metadata(table)
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/stream.py", line 108, in record_parse_metadata
    super().record_parse_metadata(table)
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 476, in record_parse_metadata
    super().record_parse_metadata(table)
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 240, in record_parse_metadata
    pos_errors = self.compute_parse_errors(table)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/torstein/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/parsers/base.py", line 185, in compute_parse_errors
    if indices[0][:2] != (-1, -1):
       ~~~~~~~^^^
IndexError: list index out of range
"""

The above exception was the direct cause of the following exception:

IndexError                                Traceback (most recent call last)
Cell In[13], line 1
----> 1 tables = get_tables(Pdf(pdf_path), "stream")

File ~/code/ai-pilot-wellreport/ai_wellreport/table_extractor.py:54, in get_tables(pdf, flavor)
     47 page_heights = pdf.get_heights()
     48 # TODO
     49 # try and compare:
     50 # flavor="lattice" (default)
     51 # flavor="stream"
     52 # flavor="network"
     53 # flavor="hybrid"
---> 54 tables = pypdf_table_extraction.read_pdf(  # type: ignore
     55     str(pdf.path),
     56     flavor=flavor,
     57     pages=pdf.page_numbers_str,
     58     strip_text="\n",
     59     parallel=True,
     60 )
     61 for table in tables:
     62     df = prep_dataframe(table.df)

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/io.py:134, in read_pdf(filepath, pages, password, flavor, suppress_stdout, parallel, layout_kwargs, debug, **kwargs)
    132 p = PDFHandler(filepath, pages=pages, password=password, debug=debug)
    133 kwargs = remove_extra(kwargs, flavor=flavor)
--> 134 tables = p.parse(
    135     flavor=flavor,
    136     suppress_stdout=suppress_stdout,
    137     parallel=parallel,
    138     layout_kwargs=layout_kwargs,
    139     **kwargs
    140 )
    141 return tables

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/handlers.py:233, in PDFHandler.parse(self, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    230             jobs.append(j)
    232         for j in jobs:
--> 233             t = j.get()
    234             tables.extend(t)
    235 else:

File ~/.pixi/envs/pipx/lib/python3.12/multiprocessing/pool.py:774, in ApplyResult.get(self, timeout)
    772     return self._value
    773 else:
--> 774     raise self._value

IndexError: list index out of range

[bosd]

Thanks for your feedback. Battle testing this thing. Hoping to get soon to a stable state for the first Major release.

File ~/code/ai-pilot-wellreport/.venv/lib/python3.12/site-packages/camelot/core.py:733, in Table.set_border(self)
    731     self.cells[index][len(self.cols) - 1].right = True
    732 for index, _col in enumerate(self.cols):
--> 733     self.cells[0][index].top = True
    734     self.cells[len(self.rows) - 1][index].bottom = True
    735 return self

IndexError: list index out of range

That should be fixed with #247

[bosd]

Describe the bug

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/core.py:443, in Table.set_edges(self, vertical, horizontal, joint_tol)
    441     i = len(self.rows) - 1
    442     for j in range(start, end):
--> 443         self.cells[i][j].bottom = True
    444 elif i == 0:  # only top edge
    445     for j in range(start, end):

IndexError: list index out of range

Steps to reproduce the bug

Expected behavior

Code

    tables_li = pypdf_table_extraction.read_pdf(
        str(pdf_path),
        pages=page_numbers,
        strip_text="\n",
    )

PDF

Cannot add, confidential

Screenshots

Environment

• OS: [e.g. macOS]
• Python version: 3.12
• Numpy version: '1.26.4'
• OpenCV version:
• Ghostscript version: GPL Ghostscript 10.02.1 (2023-11-01)
• pypdf_table_extraction version: 0.0.2

Additional context

Full traceback

2024-10-21T09:47:14 - INFO - Processing page-149
Processing page-149
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
File ~/code/ai-pilot-wellreport/table_extraction/no_conda/extraction.py:175
    171         page_to_tables[table.page].append(TableInfo(table._bbox[1], table._bbox[3], df))
    172     return dict(page_to_tables)
--> 175 page_to_tables = get_tables(pdf_path, page_numbers)
    176 markdown_content = pdf_to_markdown(pdf_path, page_numbers, page_to_tables)
    177 print(markdown_content)

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/extraction.py:163, in get_tables(pdf_path, page_numbers)
    161 elif isinstance(page_numbers, list):
    162     page_numbers = ",".join(map(str, page_numbers))
--> 163 tables_li = camelot.read_pdf(
    164     str(pdf_path),
    165     pages=page_numbers,
    166     strip_text="\n",
    167 )
    168 page_to_tables = defaultdict(list)
    169 for table in tables_li:

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/io.py:125, in read_pdf(filepath, pages, password, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    123 p = PDFHandler(filepath, pages=pages, password=password)
    124 kwargs = remove_extra(kwargs, flavor=flavor)
--> 125 tables = p.parse(
    126     flavor=flavor,
    127     suppress_stdout=suppress_stdout,
    128     parallel=parallel,
    129     layout_kwargs=layout_kwargs,
    130     **kwargs
    131 )
    132 return tables

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/handlers.py:202, in PDFHandler.parse(self, flavor, suppress_stdout, parallel, layout_kwargs, **kwargs)
    200     else:
    201         for p in self.pages:
--> 202             t = self._parse_page(
    203                 p, tempdir, parser, suppress_stdout, layout_kwargs
    204             )
    205             tables.extend(t)
    207 return TableList(sorted(tables))

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/handlers.py:232, in PDFHandler._parse_page(self, page, tempdir, parser, suppress_stdout, layout_kwargs)
    230 self._save_page(self.filepath, page, tempdir)
    231 page_path = os.path.join(tempdir, f"page-{page}.pdf")
--> 232 tables = parser.extract_tables(
    233     page_path, suppress_stdout=suppress_stdout, layout_kwargs=layout_kwargs
    234 )
    235 return tables

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/parsers/lattice.py:412, in Lattice.extract_tables(self, filename, suppress_stdout, layout_kwargs)
    408 for table_idx, tk in enumerate(
    409     sorted(self.table_bbox.keys(), key=lambda x: x[1], reverse=True)
    410 ):
    411     cols, rows, v_s, h_s = self._generate_columns_and_rows(table_idx, tk)
--> 412     table = self._generate_table(table_idx, cols, rows, v_s=v_s, h_s=h_s)
    413     table._bbox = tk
    414     _tables.append(table)

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/parsers/lattice.py:334, in Lattice._generate_table(self, table_idx, cols, rows, **kwargs)
    332 table = Table(cols, rows)
    333 # set table edges to True using ver+hor lines
--> 334 table = table.set_edges(v_s, h_s, joint_tol=self.joint_tol)
    335 # set table border edges to True
    336 table = table.set_border()

File ~/code/ai-pilot-wellreport/table_extraction/no_conda/.venv/lib/python3.12/site-packages/camelot/core.py:443, in Table.set_edges(self, vertical, horizontal, joint_tol)
    441     i = len(self.rows) - 1
    442     for j in range(start, end):
--> 443         self.cells[i][j].bottom = True
    444 elif i == 0:  # only top edge
    445     for j in range(start, end):

IndexError: list index out of range

This one should be fixed with #249

[bosd]

@tsoernes Can you please check again with the latest code from this repo?