diff --git a/src/back_reporting.py b/src/back_reporting.py index e6112ab..ad16795 100644 --- a/src/back_reporting.py +++ b/src/back_reporting.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from typing import Callable from tqdm import tqdm from multiprocessing import Pool, cpu_count +import os TOP_PATH: Path = Path(r"\\leafnow.com\shared\Accounting\CASH APPS\2023") @@ -48,10 +49,10 @@ def append_to_consolidated_report( report_path: Path, report_df: DataFrame, shee """ report_month: Path = extract_date_path(report_path) - report_name: str = f"{str(report_month.name).replace('.','-')}_ConsolidatedReport.xlsx" + report_name: str = f"{str(report_month.name).replace('.','-')}_{sheet_name}_ConsolidatedReport.xlsx" - save_path = report_month / Path(report_name) + save_path = Path("../2023",report_name) logger.debug(f"{save_path=}") # Check if the current month has a consolidated report @@ -86,9 +87,7 @@ def process_report(file: Path, extract_inst: ExtractInstruction) -> bool: report_str: str = f.read() #logger.debug(f"{report_str}") try: - df: DataFrame = extract_inst.extract_method(report_str, - Path(file.parent ,f"BACK_REPORT_{extract_inst.sheet_name}.xlsx") - ) + df: DataFrame = extract_inst.extract_method(report_str, None) if df.empty: raise ValueError("Dataframe is empty!") except Exception as e: @@ -168,13 +167,14 @@ if __name__ == "__main__": ]), ] - for folder in tqdm(FOLDERS): - try: - process_folder(folder) - print(f"Completed: {folder.folder_name}") - except Exception as e: - print(f"Failed to process {folder.folder_name} \n {e}") - continue + with Pool(cpu_count()) as pool: + for folder in tqdm(pool.imap_unordered(process_folder,FOLDERS)): + try: + process_folder(folder) + print(f"Completed: {folder.folder_name}") + except Exception as e: + print(f"Failed to process {folder.folder_name} \n {e}") + continue input("Complete!") except Exception as e: diff --git a/src/il_reports.py b/src/il_reports.py index 5867d1f..a6a4a7e 100644 --- a/src/il_reports.py +++ b/src/il_reports.py @@ -192,7 +192,7 @@ COMMON REGEX COMPONENTS """ -def ach(report: str, save_name: str): +def ach(report: str, save_name: str|None): debug(f"ACH Report {save_name} :\n{report}") lines = report.splitlines() extracted_data_dict = { @@ -246,11 +246,12 @@ def ach(report: str, save_name: str): dataframe: DataFrame = DataFrame(extracted_data_dict) # We're creating two sheets: data & summary so we need to open and excel writer # This also helps with a bug caused by larger dataframes - with pd.ExcelWriter(save_name) as writer: - debug(f"ACH: Saving data as: {save_name}") - dataframe.to_excel(writer, index=False, sheet_name="data") - # The batches dictioanry is converted to a dataframe and added as it's own sheet - DataFrame(batches).to_excel(writer, index=False, sheet_name="Summary") + if save_name is not None: + with pd.ExcelWriter(save_name) as writer: + debug(f"ACH: Saving data as: {save_name}") + dataframe.to_excel(writer, index=False, sheet_name="data") + # The batches dictioanry is converted to a dataframe and added as it's own sheet + DataFrame(batches).to_excel(writer, index=False, sheet_name="Summary") return dataframe def disposition(report: str, save_name: str): @@ -625,7 +626,7 @@ def net_invest_trial_balance(report: str, save_name: str): writer, index=True, sheet_name="Summary") return dataframe -def lockbox(report: str, save_name: str): +def lockbox(report: str, save_name: str|None): debug(f"LockBox Report {save_name}:\n{report}") lines = report.splitlines() extracted_data_dict = { @@ -678,7 +679,8 @@ def lockbox(report: str, save_name: str): extracted_data_dict["CUST NAME"].append(lines[index+1].strip()) dataframe = DataFrame(extracted_data_dict) debug(f"LockBox | Saving dataframe: {save_name}") - dataframe.to_excel(save_name, index=False) + if save_name is not None: + dataframe.to_excel(save_name, index=False) return dataframe @@ -717,7 +719,7 @@ def minv(report: str, save_name: str): # Good for PUB_WIRES, VMCC, PBP_EPAY, returned check -def payment_transactions(report: str, save_name: str): +def payment_transactions(report: str, save_name: str|None): debug(f"PayTrans | {save_name}:\n{report}") lines = report.splitlines() data_extractor = create_line_divider([6,33,52,62,80,89,110,121]) @@ -754,8 +756,9 @@ def payment_transactions(report: str, save_name: str): extracted_data_dict['INV NO'].append(inv_no) dataframe = DataFrame(extracted_data_dict) debug(f"PayTrans | Complted Dataframe:\n{dataframe}") - dataframe.to_excel(save_name, index=False) - debug(f"PayTrans | Saved to {save_name}") + if save_name is not None: + dataframe.to_excel(save_name, index=False) + debug(f"PayTrans | Saved to {save_name}") return dataframe