i had a similar problem:
i wanted to parse an incomplete (partially downloaded) sqlite3 database file.
i came up with a custom sqlite3 parser, written in python, based on kaitai_struct
example use:
prepare.sh
sqlite3 example.db '
create table t1 (a INT);
create table t2 (a INT);
insert into t1 values (1);
insert into t2 values (2);
'
du -b example.db
# 12288 example.db
sqlite3 example.db 'pragma page_size'
# 4096
expr 12288 / 4096
# 3
# remove page 3
dd if=example.db of=example-partial.db bs=4096 count=2
sqlite3 example-partial.db 'select * from t1'
# Error: in prepare, database disk image is malformed (11)
parse.py
import pysqlite3
database = "example-partial.db"
con = pysqlite3.connect(database, allow_bad_size=True)
print(f"page size: {con._db.header.page_size} bytes")
print(f"db size: {con._db.header.num_pages} pages")
print(f"idx_lock_byte_page: {con._db.header.idx_lock_byte_page}")
print(f"idx_first_ptrmap_page: {con._db.header.idx_first_ptrmap_page}")
print(f"idx_last_ptrmap_page: {con._db.header.idx_last_ptrmap_page}")
print(f"con._db.pages =", con._db.pages)
print("tables =", con._tables)
print("con._columns")
for table in con._tables:
try:
print(f"table {table}: columns =", con._columns(table))
except NotImplementedError as err:
print("ignoring NotImplementedError:", err)
def format_values(values):
result = []
max_len = 50
for val in values:
s = repr(val)
if len(s) > max_len:
s = s[0:max_len] + "..."
result.append(s)
return "[" + ", ".join(result) + "]"
print("con._table_values")
for table in con._tables:
print(f"con._table_values(table={repr(table)})")
num_rows = 0
for row_id, values in enumerate(con._table_values(table)):
num_rows += 1
print(f"table {table}: row {row_id + 1} =", format_values(values))
output of parse.py
warning: bad size. expected 12288. actual 8192
page size: 4096 bytes
db size: 3 pages
idx_lock_byte_page: 262144
idx_first_ptrmap_page: 0
idx_last_ptrmap_page: -1
con._db.pages = <pysqlite3.parser.sqlite3_helpers.PagesList object at 0x7fe44a4ee230>
tables = ['t1', 't2']
con._columns
table t1: columns = ['a']
table t2: columns = ['a']
con._table_values
con._table_values(table='t1')
table t1: row 1 = [1]
con._table_values(table='t2')
EOFError: requested 1 bytes, but only 0 bytes available
the EOFError
is expected, because page 3 is missing, which contains the table t2