From b3f1fc667aae87352fa7b5070b6130f187564cc2 Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Tue, 22 Aug 2023 21:04:59 +1000 Subject: [PATCH] IfcCSV can now create summary rows (e.g. total, average) at the end of the table (may or may not be in addition to group totals). --- .../blenderbim/bim/module/csv/operator.py | 8 +- .../blenderbim/bim/module/csv/prop.py | 10 + .../blenderbim/bim/module/csv/ui.py | 9 +- src/ifccsv/ifccsv.py | 184 +++++++++++------- 4 files changed, 141 insertions(+), 70 deletions(-) diff --git a/src/blenderbim/blenderbim/bim/module/csv/operator.py b/src/blenderbim/blenderbim/bim/module/csv/operator.py index db1d6cee52..ee3e312e29 100644 --- a/src/blenderbim/blenderbim/bim/module/csv/operator.py +++ b/src/blenderbim/blenderbim/bim/module/csv/operator.py @@ -82,6 +82,7 @@ class ImportCsvAttributes(bpy.types.Operator): new.header = attribute["header"] new.sort = attribute["sort"] new.group = attribute["group"] + new.summary = attribute["summary"] return {"FINISHED"} def invoke(self, context, event): @@ -104,7 +105,8 @@ class ExportCsvAttributes(bpy.types.Operator): data = { "query": tool.Search.export_filter_query(props.filter_groups), "attributes": [ - {"name": a.name, "header": a.header, "sort": a.sort, "group": a.group} for a in props.csv_attributes + {"name": a.name, "header": a.header, "sort": a.sort, "group": a.group, "summary": a.summary} + for a in props.csv_attributes ], } @@ -152,11 +154,14 @@ class ExportIfcCsv(bpy.types.Operator): sort = [] groups = [] + summaries = [] for attribute in props.csv_attributes: if attribute.sort != "NONE": sort.append({"name": attribute.name, "order": attribute.sort}) if attribute.group != "NONE": groups.append({"name": attribute.name, "type": attribute.group, "varies_value": attribute.varies_value}) + if attribute.summary != "NONE": + summaries.append({"name": attribute.name, "type": attribute.summary}) sep = props.csv_custom_delimiter if props.csv_delimiter == "CUSTOM" else props.csv_delimiter ifc_csv.export( @@ -174,6 +179,7 @@ class ExportIfcCsv(bpy.types.Operator): bool_false=props.false_value, sort=sort, groups=groups, + summaries=summaries, ) return {"FINISHED"} diff --git a/src/blenderbim/blenderbim/bim/module/csv/prop.py b/src/blenderbim/blenderbim/bim/module/csv/prop.py index 86b32d2b15..96fc51e822 100644 --- a/src/blenderbim/blenderbim/bim/module/csv/prop.py +++ b/src/blenderbim/blenderbim/bim/module/csv/prop.py @@ -49,6 +49,15 @@ class CsvAttribute(PropertyGroup): ] ) varies_value: StringProperty(default="Varies", name="Varies Value") + summary: EnumProperty( + items=[ + ("NONE", "None", ""), + ("SUM", "Sum", "Sums the total value of all rows."), + ("AVERAGE", "Average", "Averages the total value of all rows."), + ("MIN", "Min", "Gets the minimum value of all rows."), + ("MAX", "Max", "Gets the maximum value of all rows."), + ] + ) class CsvProperties(PropertyGroup): @@ -96,4 +105,5 @@ class CsvProperties(PropertyGroup): should_show_settings: BoolProperty(default=False, name="Show Settings") should_show_sort: BoolProperty(default=False, name="Show Sorting") should_show_group: BoolProperty(default=False, name="Show Grouping") + should_show_summary: BoolProperty(default=False, name="Show Summary") should_load_from_memory: BoolProperty(default=False, name="Load from Memory") diff --git a/src/blenderbim/blenderbim/bim/module/csv/ui.py b/src/blenderbim/blenderbim/bim/module/csv/ui.py index fb548a727e..d4f4951155 100644 --- a/src/blenderbim/blenderbim/bim/module/csv/ui.py +++ b/src/blenderbim/blenderbim/bim/module/csv/ui.py @@ -47,8 +47,6 @@ class BIM_PT_ifccsv(Panel): row.alignment = "RIGHT" row.operator("bim.import_csv_attributes", icon="IMPORT", text="") row.operator("bim.export_csv_attributes", icon="EXPORT", text="") - row.prop(props, "should_show_sort", icon="SORTSIZE", text="") - row.prop(props, "should_show_group", icon="OUTLINER_COLLECTION", text="") row.prop(props, "should_show_settings", icon="PREFERENCES", text="") if not IfcStore.get_file() or not props.should_load_from_memory: @@ -85,17 +83,22 @@ class BIM_PT_ifccsv(Panel): row = layout.row(align=True) row.operator("bim.add_csv_attribute", icon="ADD") + row.prop(props, "should_show_sort", icon="SORTSIZE", text="") + row.prop(props, "should_show_group", icon="OUTLINER_COLLECTION", text="") + row.prop(props, "should_show_summary", icon="SYNTAX_ON", text="") for index, attribute in enumerate(props.csv_attributes): row = layout.row(align=True) row.prop(attribute, "name", text="") + row.prop(attribute, "header", text="") if props.should_show_sort: row.prop(attribute, "sort", text="") if props.should_show_group: row.prop(attribute, "group", text="") if attribute.group == "VARIES": row.prop(attribute, "varies_value", text="") - row.prop(attribute, "header", text="") + if props.should_show_summary: + row.prop(attribute, "summary", text="") row.operator("bim.remove_csv_attribute", icon="X", text="").index = index row = layout.row(align=True) diff --git a/src/ifccsv/ifccsv.py b/src/ifccsv/ifccsv.py index 9849227b99..b99288ca4b 100755 --- a/src/ifccsv/ifccsv.py +++ b/src/ifccsv/ifccsv.py @@ -74,6 +74,7 @@ class IfcCsv: bool_false="NO", sort=None, groups=None, + summaries=None, ): self.ifc_file = ifc_file self.results = [] @@ -112,65 +113,119 @@ class IfcCsv: else: self.headers.append(attribute) - if groups: - group_results = {} - group_indices = {} - group_values = {} - group_varies_values = {} + self.group_results(groups, attributes) + self.summarise_results(summaries, attributes) + self.sort_results(sort, attributes) - for group in groups: - index = attributes.index(group["name"]) - group_indices.setdefault(group["type"], []) - group_indices[group["type"]].append(index) - if group["type"] == "VARIES": - group_varies_values[index] = group["varies_value"] + if format == "csv": + self.export_csv(output, delimiter=delimiter) + elif format == "ods": + self.export_ods(output, should_preserve_existing=should_preserve_existing) + elif format == "xlsx": + self.export_xlsx(output, should_preserve_existing=should_preserve_existing) + elif format == "pd": + return self.export_pd() - for row in self.results: - key = "-".join([str(row[gi]) for gi in group_indices.get("GROUP", [])]) - for group_type, gis in group_indices.items(): - if group_type in ("CONCAT", "VARIES"): - for gi in gis: - group_values.setdefault(key, {}).setdefault(gi, set()) - group_values[key][gi].add(str(row[gi])) - elif group_type in ("SUM", "AVERAGE", "MIN", "MAX"): - for gi in gis: - group_values.setdefault(key, {}).setdefault(gi, []) - try: - value = float(row[gi]) - except: - continue - group_values[key][gi].append(value) - group_results[key] = row + def group_results(self, groups, attributes): + if not groups: + return + group_results = {} + group_indices = {} + group_values = {} + group_varies_values = {} + + for group in groups: + index = attributes.index(group["name"]) + group_indices.setdefault(group["type"], []) + group_indices[group["type"]].append(index) + if group["type"] == "VARIES": + group_varies_values[index] = group["varies_value"] + + for row in self.results: + key = "-".join([str(row[gi]) for gi in group_indices.get("GROUP", [])]) for group_type, gis in group_indices.items(): - if group_type == "CONCAT": - for key, result in group_results.items(): - for gi in gis: - result[gi] = ", ".join(group_values[key][gi]) - elif group_type == "VARIES": - for key, result in group_results.items(): - for gi in gis: - if len(group_values[key][gi]) > 1: - result[gi] = group_varies_values[gi] - elif group_type == "SUM": - for key, result in group_results.items(): - for gi in gis: - result[gi] = sum(group_values[key][gi]) - elif group_type == "AVERAGE": - for key, result in group_results.items(): - for gi in gis: - result[gi] = mean(group_values[key][gi]) - elif group_type == "MIN": - for key, result in group_results.items(): - for gi in gis: - result[gi] = min(group_values[key][gi]) - elif group_type == "MAX": - for key, result in group_results.items(): - for gi in gis: - result[gi] = max(group_values[key][gi]) + if group_type in ("CONCAT", "VARIES"): + for gi in gis: + group_values.setdefault(key, {}).setdefault(gi, set()) + group_values[key][gi].add(str(row[gi])) + elif group_type in ("SUM", "AVERAGE", "MIN", "MAX"): + for gi in gis: + group_values.setdefault(key, {}).setdefault(gi, []) + try: + value = float(row[gi]) + except: + continue + group_values[key][gi].append(value) + group_results[key] = row - self.results = group_results.values() + for group_type, gis in group_indices.items(): + if group_type == "CONCAT": + for key, result in group_results.items(): + for gi in gis: + result[gi] = ", ".join(group_values[key][gi]) + elif group_type == "VARIES": + for key, result in group_results.items(): + for gi in gis: + if len(group_values[key][gi]) > 1: + result[gi] = group_varies_values[gi] + elif group_type == "SUM": + for key, result in group_results.items(): + for gi in gis: + result[gi] = sum(group_values[key][gi]) + elif group_type == "AVERAGE": + for key, result in group_results.items(): + for gi in gis: + result[gi] = mean(group_values[key][gi]) + elif group_type == "MIN": + for key, result in group_results.items(): + for gi in gis: + result[gi] = min(group_values[key][gi]) + elif group_type == "MAX": + for key, result in group_results.items(): + for gi in gis: + result[gi] = max(group_values[key][gi]) + self.results = group_results.values() + + def summarise_results(self, summaries, attributes): + self.summaries = [None] * len(attributes) + + if not summaries: + return + + summary_indices = {} + summary_values = {} + + for summary in summaries: + index = attributes.index(summary["name"]) + summary_indices.setdefault(summary["type"], []) + summary_indices[summary["type"]].append(index) + + for row in self.results: + for summary_type, sis in summary_indices.items(): + if summary_type in ("SUM", "AVERAGE", "MIN", "MAX"): + for si in sis: + summary_values.setdefault(si, []) + try: + value = float(row[si]) + except: + continue + summary_values[si].append(value) + + for summary_type, sis in summary_indices.items(): + for si in sis: + if summary_type == "SUM": + self.summaries[si] = sum(summary_values[si]) + elif summary_type == "AVERAGE": + self.summaries[si] = mean(summary_values[si]) + elif summary_type == "MIN": + self.summaries[si] = min(summary_values[si]) + elif summary_type == "MAX": + self.summaries[si] = max(summary_values[si]) + self.summaries[si] = summary_type.title() + ": " + str(self.summaries[si]) + + def sort_results(self, sort, attributes): if sort: def natural_sort(value): if isinstance(value, str): @@ -187,28 +242,24 @@ class IfcCsv: else: self.results = sorted(self.results, key=lambda x: x[1 if include_global_id else 0]) - if format == "csv": - self.export_csv(output, delimiter=delimiter) - elif format == "ods": - self.export_ods(output, should_preserve_existing=should_preserve_existing) - elif format == "xlsx": - self.export_xlsx(output, should_preserve_existing=should_preserve_existing) - elif format == "pd": - return self.export_pd() - def export_csv(self, output, delimiter=None): with open(output, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f, delimiter=delimiter) writer.writerow(self.headers) for row in self.results: writer.writerow(row) + if any([s for s in self.summaries if s is not None]): + writer.writerow(self.summaries) def export_ods(self, output, should_preserve_existing=False): + df = self.export_pd() + if self.summaries: + df.loc[df.shape[0]] = self.summaries + if os.path.exists(output) and should_preserve_existing: ods_document = load(output) first_table = ods_document.spreadsheet.getElementsByType(Table)[0] - df = self.export_pd() for col_index, col in enumerate(df.columns): # Assuming the first row of the table contains headers header_cell = self.get_col(first_table.getElementsByType(TableRow)[0], col_index) @@ -232,7 +283,6 @@ class IfcCsv: ods_document.save(output) else: - df = self.export_pd() df.to_excel(output, index=False, engine="odf") def set_cell_value(self, cell, value): @@ -265,6 +315,10 @@ class IfcCsv: return new_cell def export_xlsx(self, output, should_preserve_existing=False): + df = self.export_pd() + if self.summaries: + df.loc[df.shape[0]] = self.summaries + if os.path.exists(output): book = openpyxl.load_workbook(output) with pd.ExcelWriter( @@ -273,10 +327,8 @@ class IfcCsv: mode="a", if_sheet_exists="overlay" if should_preserve_existing else "replace", ) as writer: - df = self.export_pd() df.to_excel(writer, sheet_name=book.sheetnames[0], index=False) else: - df = self.export_pd() df.to_excel(output, index=False, engine="openpyxl") def export_pd(self):