As I am iterating through a series of .xlsm
and .xlsx
files, I am trying to copy the first sheet into a storage workbook. However, after about 30-40 files, I start to exclusively get a MemoryError
from even some very small .xlsx
files. I feel as though it shouldn't be too taxing to individually open excel files.
I believe I'm saving and closing the necessary files. But clearly I'm overloading the memory somewhere. Any ideas?
import logging
import os
import openpyxl as xl
from copy import copy
from zipfile import BadZipFile
# Create and configure logger
LOG_FORMAT = "%(levelname)s - %(message)s"
logging.basicConfig(filename="errors.log",
level=logging.INFO,
format = LOG_FORMAT,
filemode ='w')
logger = logging.getLogger()
def copy_paste_sheets(sheet_giv, sheet_rec, filename_giv='', filename_rec='', start_row=1, start_col=1, end_row=None, end_col=None, offset_row=0, offset_col=0):
if end_row == None:
end_row = sheet_giv.max_row
if end_col == None:
end_col = sheet_giv.max_column
print(f"Copying and Pasting {filename_giv} {sheet_giv} to {filename_rec}{sheet_rec}...")
for i in range(start_row, end_row + 1):
for j in range(start_col, end_col + 1):
giv_cell = sheet_giv.cell(row=i, column=j)
rec_cell = sheet_rec.cell(row=i+offset_row, column=j+offset_col, value=giv_cell.value)
if giv_cell.has_style:
rec_cell.font = copy(giv_cell.font)
rec_cell.border = copy(giv_cell.border)
rec_cell.fill = copy(giv_cell.fill)
rec_cell.number_format = copy(giv_cell.number_format)
rec_cell.protection = copy(giv_cell.protection)
rec_cell.alignment = copy(giv_cell.alignment)
def main():
directory = r'C:\path'
rec_wb = xl.Workbook()
for n, filename in enumerate(os.listdir(directory)):
if filename.endswith(".xlsx") or filename.endswith(".xlsm"):
try:
giv_wb = xl.load_workbook(filename=os.path.join(directory, filename))
giv_ws = giv_wb.worksheets[0]
rec_ws = rec_wb.create_sheet(str(n), -1)
title_cell = rec_ws.cell(row=1, column=1)
title_cell.value = filename
copy_paste_sheets(giv_ws, rec_ws, filename_giv=filename, end_row=50, end_col=15, offset_row=1)
except (BadZipFile, MemoryError) as e:
if type(e) == MemoryError:
e = 'MemoryError'
print(f'No: {n} - {filename} = {e}')
logger.error(f'No: {n} - {filename} = {e}')
giv_wb.close()
elif not filename.endswith(".xlsx") or not filename.endswith(".xlsm"):
logger.info(f'No: {n} - {filename} = Not xlsx or xlsm file')
rec_wb.save("output.xlsx")
rec_wb.close()
rec_wb = xl.load_workbook(filename="output.xlsx")
if __name__ == "__main__":
main()
giv_wb.close()
should probably be at the bottom of thetry
block, undercopy_paste_sheets