From c2bb01f6189e44dc2f92ec1a484e749119bdf9e5 Mon Sep 17 00:00:00 2001 From: IndianaEunice Date: Tue, 17 Nov 2026 13:54:11 +0300 Subject: [PATCH 1/4] mv materialized view for the data download --- .../1794820755131-CreateVaExportMv.ts | 502 ++++++++++++++++++ .../db/shared/dynamic-export.service-v2.ts | 170 ++++++ 2 files changed, 672 insertions(+) create mode 100644 src/API/src/db/migrations/1794820755131-CreateVaExportMv.ts diff --git a/src/API/src/db/migrations/1794820755131-CreateVaExportMv.ts b/src/API/src/db/migrations/1794820755131-CreateVaExportMv.ts new file mode 100644 index 000000000..c06c7b190 --- /dev/null +++ b/src/API/src/db/migrations/1794820755131-CreateVaExportMv.ts @@ -0,0 +1,502 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * Creates va_export_mv: one row per occurrence in an Approved dataset, with the + * 422 VA template columns in template order (plus occurrence_id for paging). + * + * Generated from RAW_TEMPLATE_FIELD_MAPPING (template-mapping.ts, with the mapping + * fixes applied). If the mapping changes, regenerate the SQL and add a NEW migration. + */ +export class CreateVaExportMv1794821010089 implements MigrationInterface { + name = 'CreateVaExportMv1794821010089'; + + public async up(queryRunner: QueryRunner): Promise { + // 1. Remove any earlier copy (dropping the view also drops its index) + await queryRunner.query(`DROP MATERIALIZED VIEW IF EXISTS va_export_mv`); + + // 2. Create the view, empty for now + await queryRunner.query(` +CREATE MATERIALIZED VIEW va_export_mv AS +SELECT + o.id AS occurrence_id, + o.confidentiality_status AS confidentiality_status, + o.bio_data AS bio_data, + o.adult_data AS adult_data, + o.larval_data AS larval_data, + o.abundance_data AS abundance_data, + b.larval_site_data AS larval_site_data, + o.pheno_data AS pheno_data, + o.geno_data AS geno_data, + o.insecticide_resistance_data AS insecticide_resistance_data, + o.source_id AS source_id, + r.citation AS citation_doi, + r.author AS author, + r.article_title AS article_title, + r.journal_title AS journal_title, + r.year AS publication_year, + o.personal_communication AS personal_communication, + s.country AS country, + s.site AS site, + s.latitude AS latitude_1, + s.longitude AS longitude_1, + s.latitude_2 AS latitude_2, + s.longitude_2 AS longitude_2, + s.latitude_3 AS latitude_3, + s.longitude_3 AS longitude_3, + s.latitude_4 AS latitude_4, + s.longitude_4 AS longitude_4, + s.latitude_5 AS latitude_5, + s.longitude_5 AS longitude_5, + s.latitude_6 AS latitude_6, + s.longitude_6 AS longitude_6, + s.latitude_7 AS latitude_7, + s.longitude_7 AS longitude_7, + s.latitude_8 AS latitude_8, + s.longitude_8 AS longitude_8, + s.confidence_in_georef AS confidence_in_georef, + s.area_type AS area_type, + s.georef_source AS georef_source, + s.admin_level_1 AS admin_level_1, + s.admin_level_2 AS admin_level_2, + b.insecticide_control AS insecticide_control, + sa.control_type AS control_type, + b.itn_use AS itn_use, + sa.sampling_occurrence_1 AS sampling_occurrence_1, + sa.occurrence_n_1 AS occurrence_n_1, + sa.sampling_occurrence_2 AS sampling_occurrence_2, + sa.occurrence_n_2 AS occurrence_n_2, + sa.sampling_occurrence_3 AS sampling_occurrence_3, + sa.occurrence_n_3 AS occurrence_n_3, + sa.sampling_occurrence_4 AS sampling_occurrence_4, + sa.occurrence_n_4 AS occurrence_n_4, + sa.occurrence_n_total AS occurrence_n_total, + o.binary_presence AS binary_presence, + o.binary_absence AS binary_absence, + b.month_start AS month_start, + b.month_end AS month_end, + b.year_start AS year_start, + b.year_end AS year_end, + b.season_given AS season_given, + b.season_calc AS season_calc, + b.rainfall_time AS rainfall_time, + rs.species AS species, + rs.species_id_1 AS species_id_1, + rs.species_id_2 AS species_id_2, + env.roof AS roof, + env.walls AS walls, + env.house_screening AS house_screening, + env.open_eaves AS open_eaves, + env.cooking AS cooking, + env.common_occupation_1 AS common_occupation_1, + env.common_occupation_2 AS common_occupation_2, + env.common_occupation_3 AS common_occupation_3, + env.outdoor_activities_at_night AS outdoor_activities_at_night, + env.sleeping_outdoors AS sleeping_outdoors, + env.outdoor_timings_hours AS outdoor_timings_hours, + env.average_bedtime AS average_bedtime, + env.average_wake_time AS average_wake_time, + env.time_people_leave_home_in_morning AS time_people_leave_home_in_morning, + env.hours_spent_away_from_home_per_day AS hours_spent_away_from_home_per_day, + env.seasonal_labour AS seasonal_labour, + env.forest AS forest, + env.farming AS farming, + env.livestock_1 AS livestock_1, + env.livestock_2 AS livestock_2, + env.livestock_3 AS livestock_3, + env.livestock_4 AS livestock_4, + env.local_plants AS local_plants, + bl.sampling_biology_1 AS sampling_biology_1, + bl.sampling_biology_2 AS sampling_biology_2, + bl.sampling_biology_3 AS sampling_biology_3, + bl.sampling_biology_n AS sampling_biology_n, + bl.parity_n AS parity_n, + bl.parity_total AS parity_total, + bl.parity_percent AS parity_percent, + bl.daily_survival_rate_percent AS daily_survival_rate_percent, + bl.fecundity_mean_batch_size AS fecundity_mean_batch_size, + bl.gonotrophic_cycle_days AS gonotrophic_cycle_days, + inf.sampling_infetion_1 AS sampling_infection_1, + inf.sampling_infection_2 AS sampling_infection_2, + inf.sampling_infection_3 AS sampling_infection_3, + inf.sampling_infection_n AS sampling_infection_n, + inf.sporozoite_rate_by_dissection_n AS sporozoite_rate_by_dissection_n, + inf.sporozoite_rate_by_dissection_total AS sporozoite_rate_by_dissection_total, + inf.sporozoite_rate_by_dissection_percent AS sporozoite_rate_by_dissection_percent, + inf.sporozoite_rate_by_csp_n_pool AS sporozoite_rate_by_csp_n_pool, + inf.sporozoite_rate_by_csp_total_pool AS sporozoite_rate_by_csp_total_pool, + inf.sporozoite_rate_by_csp_percent AS sporozoite_rate_by_csp_percent, + inf.sporozoite_rate_p_falciparum_n AS sporozoite_rate_p_falciparum_n, + inf.sporozoite_rate_p_falciparum_total AS sporozoite_rate_p_falciparum_total, + inf.sporozoite_rate_by_p_falciparum_percent AS sporozoite_rate_p_falciparum_percent, + inf.sporozoite_rate_p_vivax_n AS sporozoite_rate_p_vivax_n, + inf.sporozoite_rate_p_vivax_total AS sporozoite_rate_p_vivax_total, + inf.sporozoite_rate_by_p_vivax_percent AS sporozoite_rate_p_vivax_percent, + inf.oocyst_n AS oocyst_n, + inf.oocyst_total AS oocyst_total, + inf.oocyst_rate_percent AS oocyst_rate_percent, + inf.eir AS eir, + inf.eir_days AS eir_period, + NULL AS ext_incubation_period_days, + br.hbr_sampling_indoor AS hbr_sampling_indoor, + br.indoor_hbr AS indoor_hbr, + br.hbr_sampling_outdoor AS hbr_sampling_outdoor, + br.outdoor_hbr AS outdoor_hbr, + br.hbr_sampling_combined_1 AS hbr_sampling_combined_1, + br.hbr_sampling_combined_2 AS hbr_sampling_combined_2, + br.hbr_sampling_combined_3 AS hbr_sampling_combined_3, + br.hbr_sampling_combined_n AS hbr_sampling_combined_n, + br.combined_hbr AS combined_hbr, + br.hbr_unit AS hbr_unit, + br.abr_sampling_1 AS abr_sampling_1, + br.abr_sampling_2 AS abr_sampling_2, + br.abr_sampling_3 AS abr_sampling_3, + br.abr_sampling_n AS abr_sampling_n, + br.abr AS abr, + br.abr_unit AS abr_unit, + az.host_sampling_indoor AS host_sampling_indoor, + az.indoor_host_n AS indoor_host_n, + az.indoor_host_total AS indoor_host_total, + az.indoor_host_perc AS indoor_host_percent, + az.host_sampling_outdoor AS host_sampling_outdoor, + az.outdoor_host_n AS outdoor_host_n, + az.outdoor_host_total AS outdoor_host_total, + az.outdoor_host_percent AS outdoor_host_percent, + az.host_sampling_combined_1 AS host_sampling_combined_1, + az.host_sampling_combined_2 AS host_sampling_combined_2, + az.host_sampling_combined_3 AS host_sampling_combined_3, + az.host_sampling_combined_n AS host_sampling_combined_n, + az.combined_host_n AS combined_host_n, + az.combined_host_total AS combined_host_total, + az.combined_host AS combined_host, + az.host_unit AS host_unit, + az.host_sampling_other_1 AS host_sampling_other_1, + az.host_sampling_other_2 AS host_sampling_other_2, + az.host_sampling_other_3 AS host_sampling_other_3, + az.host_sampling_other_n AS host_sampling_other_n, + az.other_host_n AS other_host_n, + az.other_host_total AS other_host_total, + az.host_other AS host_other, + az.host_other_unit AS host_other_unit, + eg.biting_number_of_sampling_nights_indoors AS biting_number_of_sampling_nights_indoors, + eg.biting_sampling_indoor AS biting_sampling_indoor, + eg.indoor_biting_n AS indoor_biting_n, + eg.indoor_biting_total AS indoor_biting_total, + eg.indoor_biting_data AS indoor_biting_data, + eg.biting_number_of_sampling_nights_outdoors AS biting_number_of_sampling_nights_outdoors, + eg.biting_sampling_outdoor AS biting_sampling_outdoor, + eg.outdoor_biting_n AS outdoor_biting_n, + eg.outdoor_biting_total AS outdoor_biting_total, + eg.outdoor_biting_data AS outdoor_biting_data, + eg.indoor_outdoor_biting_unit AS indoor_outdoor_biting_unit, + ba.biting_activity_indoor_number_of_sampling_nights AS biting_activity_indoor_number_of_sampling_nights, + ba."1800_1900_in" AS "X1800_1900_in", + ba."1900_2000_in" AS "X1900_2000_in", + ba."2000_2100_in" AS "X2000_2100_in", + ba."2100_2200_in" AS "X2100_2200_in", + ba."2200_2300_in" AS "X2200_2300_in", + ba."2300_0000_in" AS "X2300_0000_in", + ba."0000_0100_in" AS "X0000_0100_in", + ba."0100_0200_in" AS "X0100_0200_in", + ba."0200_0300_in" AS "X0200_0300_in", + ba."0300_0400_in" AS "X0300_0400_in", + ba."0400_0500_in" AS "X0400_0500_in", + ba."0500_0600_in" AS "X0500_0600_in", + ba.biting_activity_outdoor_number_of_sampling_nights AS biting_activity_outdoor_number_of_sampling_nights, + ba."1800_1900_out" AS "X1800_1900_out", + ba."1900_2000_out" AS "X1900_2000_out", + ba."2000_2100_out" AS "X2000_2100_out", + ba."2100_2200_out" AS "X2100_2200_out", + ba."2200_2300_out" AS "X2200_2300_out", + ba."2300_0000_out" AS "X2300_0000_out", + ba."0000_0100_out" AS "X0000_0100_out", + ba."0100_0200_out" AS "X0100_0200_out", + ba."0200_0300_out" AS "X0200_0300_out", + ba."0300_0400_out" AS "X0300_0400_out", + ba."0400_0500_out" AS "X0400_0500_out", + ba."0500_0600_out" AS "X0500_0600_out", + ba.biting_activity_combined_number_of_sampling_nights AS biting_activity_combined_number_of_sampling_nights, + ba."1800_1900_combined" AS "X1800_1900_combined", + ba."1900_2000_combined" AS "X1900_2000_combined", + ba."2000_2100_combined" AS "X2000_2100_combined", + ba."2100_2200_combined" AS "X2100_2200_combined", + ba."2200_2300_combined" AS "X2200_2300_combined", + ba."2300_0000_combined" AS "X2300_0000_combined", + ba."0000_0100_combined" AS "X0000_0100_combined", + ba."0100_0200_combined" AS "X0100_0200_combined", + ba."0200_0300_combined" AS "X0200_0300_combined", + ba."0300_0400_combined" AS "X0300_0400_combined", + ba."0400_0500_combined" AS "X0400_0500_combined", + ba."0500_0600_combined" AS "X0500_0600_combined", + ey.resting_sampling_indoor AS resting_sampling_indoor, + ey.unfed_indoor AS unfed_indoor, + ey.fed_indoor AS fed_indoor, + ey.gravid_indoor AS gravid_indoor, + ey.total_indoor AS total_indoor, + ey.resting_sampling_outdoor AS resting_sampling_outdoor, + ey.unfed_outdoor AS unfed_outdoor, + ey.fed_outdoor AS fed_outdoor, + ey.gravid_outdoor AS gravid_outdoor, + ey.total_outdoor AS total_outdoor, + ey.resting_sampling_other AS resting_sampling_other, + ey.unfed_other AS unfed_other, + ey.fed_other AS fed_other, + ey.gravid_other AS gravid_other, + ey.total_other AS total_other, + ey.resting_unit AS resting_unit, + ls.larval_instars_found_1 AS larval_instars_found_1, + ls.larval_habitat_1 AS larval_habitat_1, + ls.larval_site_character_1 AS larval_site_character_1, + ls.larval_turbidity_1 AS larval_turbidity_1, + ls.larval_salinity_1 AS larval_salinity_1, + ls.larval_vegetation_1 AS larval_vegetation_1, + ls.larval_shade_1 AS larval_shade_1, + ls.larval_water_current_1 AS larval_water_current_1, + ls.larval_size_1 AS larval_size_1, + ls.larval_depth_1 AS larval_depth_1, + ls.larval_permanence_1 AS larval_permanence_1, + ls.larval_other_fauna_1 AS larval_other_fauna_1, + ls.larval_control_present_1 AS larval_control_present_1, + ls.larval_instars_found_2 AS larval_instars_found_2, + ls.larval_habitat_2 AS larval_habitat_2, + ls.larval_site_character_2 AS larval_site_character_2, + ls.larval_turbidity_2 AS larval_turbidity_2, + ls.larval_salinity_2 AS larval_salinity_2, + ls.larval__vegetation_2 AS larval__vegetation_2, + ls.larval_shade_2 AS larval_shade_2, + ls.larval_water_current_2 AS larval_water_current_2, + ls.larval_size_2 AS larval_size_2, + ls.larval_depth_2 AS larval_depth_2, + ls.larval_permanence_2 AS larval_permanence_2, + ls.larval_other_fauna_2 AS larval_other_fauna_2, + ls.larval_control_present_2 AS larval_control_present_2, + ls.larval_instars_found_3 AS larval_instars_found_3, + ls.larval_habitat_3 AS larval_habitat_3, + ls.larval_site_character_3 AS larval_site_character_3, + ls.larval_turbidity_3 AS larval_turbidity_3, + ls.larval_salinity_3 AS larval_salinity_3, + ls.larval_vegetation_3 AS larval_vegetation_3, + ls.larval_shade_3 AS larval_shade_3, + ls.larval_water_current_3 AS larval_water_current_3, + ls.larval_size_3 AS larval_size_3, + ls.larval_depth_3 AS larval_depth_3, + ls.larval_permanence_3 AS larval_permanence_3, + ls.larval_other_fauna_3 AS larval_other_fauna_3, + ls.larval_control_present_3 AS larval_control_present_3, + ir.bioassay_representative_of_complex_at_site AS bioassay_representative_of_complex_at_site, + ir.bioassay_repr_complex_site_disagg_no_adj AS bioassay_represents_complex_if_disaggregated_values_combined, + ir.generation AS generation, + ir.wild_caught_larvae_or_adults AS wild_caught_larvae_or_adults, + ir.lower_age_days AS lower_age_days, + ir.upper_age_days AS upper_age_days, + ir.test_protocol AS test_protocol, + ir.insecticide_tested AS insecticide_tested, + ir.insecticide_class AS insecticide_class, + ir.irac_moa AS irac_moa, + ir.irac_moa_code AS irac_moa_code, + ir.concentration_percent AS concentration_percent, + ir.concentration_micrograms AS concentration_micrograms, + ir.exposure_period_min AS exposure_period_min, + ir.intensity_multiplier AS intensity_multiplier, + ir.synergist_tested AS synergist_tested, + ir.synergist_concentration AS synergist_concentration, + ir.synergist_concentration_unit AS synergist_concentration_unit, + ir.mosquitoes_tested_n AS mosquitoes_tested_n, + ir.mosquitoes_dead_n AS mosquitoes_dead_n, + ir.percent_mortality AS percent_mortality, + ir.knock_down_exposure_time_min AS knock_down_exposure_time_min, + ir.mosquitoes_knocked_down_n AS mosquitoes_knocked_down_n, + ir.knock_down_percent AS knock_down_percent, + ir.kdt_50_percent_min AS kdt_50_percent_min, + ir.kdt_90_percent_min AS kdt_90_percent_min, + ir.kdt_95_percent_min AS kdt_95_percent_min, + gr.genotypic_test_representative_of_species_at_site AS genotypic_test_representative_of_species_at_site, + gr.genotypic_repr_species_site_disagg_no_adj AS genotypic_represents_species_if_disaggregated_values_combined, + gr.minor_species_missing_allele_frequency_data AS minor_species_missing_allele_frequency_data, + gr.genotypic_sample_first_been_through_bioassay_tests AS genotypic_sample_first_been_through_bioassay_tests, + gr.genotypic_sample_linked_to_a_specific_bioassay AS genotypic_sample_linked_to_a_specific_bioassay, + gr.bioassay_subsample_used_in_genotypic_test AS bioassay_subsample_used_in_genotypic_test, + vms.vgsc_method_1 AS vgsc_method_1, + vms.vgsc_method_2 AS vgsc_method_2, + vms.vgsc_number_of_mosquitoes_tested AS vgsc_number_of_mosquitoes_tested, + vms.vgsc_generation AS vgsc_generation, + vgf.vgsc995l_vgsc995l_n AS vgsc995l_vgsc995l_n, + vgf.vgsc995l_vgsc995l_percent AS vgsc995l_vgsc995l_percent, + vgf.vgsc995l_vgsc995f_n AS vgsc995l_vgsc995f_n, + vgf.vgsc995l_vgsc995f_percent AS vgsc995l_vgsc995f_percent, + vgf.vgsc995f_vgsc995f_n AS vgsc995f_vgsc995f_n, + vgf.vgsc995f_vgsc995f_percent AS vgsc995f_vgsc995f_percent, + vgf.vgsc995l_vgsc995s_n AS vgsc995l_vgsc995s_n, + vgf.vgsc995l_vgsc995s_percent AS vgsc995l_vgsc995s_percent, + vgf.vgsc995s_vgsc995s_n AS vgsc995s_vgsc995s_n, + vgf.vgsc995s_vgsc995s_percent AS vgsc995s_vgsc995s_percent, + vgf.vgsc995l_vgsc995c_n AS vgsc995l_vgsc995c_n, + vgf.vgsc995l_vgsc995c_percent AS vgsc995l_vgsc995c_percent, + vgf.vgsc995c_vgsc995c_n AS vgsc995c_vgsc995c_n, + vgf.vgsc995c_vgsc995c_percent AS vgsc995c_vgsc995c_percent, + vgf.null_vgsc995c_or_vgsc995c_vgsc995c_n AS null_vgsc995c_or_vgsc995c_vgsc995c_n, + vgf.null_vgsc995c_or_vgsc995c_vgsc995c_percent AS null_vgsc995c_or_vgsc995c_vgsc995c_percent, + vgf.vgsc995f_vgsc995s_n AS vgsc995f_vgsc995s_n, + vgf.vgsc995f_vgsc995s_percent AS vgsc995f_vgsc995s_percent, + vgf.vgsc995f_vgsc995c_n AS vgsc995f_vgsc995c_n, + vgf.vgsc995f_vgsc995c_percent AS vgsc995f_vgsc995c_percent, + kdr.susceptible_susceptible_n AS susceptible_susceptible_n, + kdr.susceptible_susceptible_percent AS susceptible_susceptible_percent, + kdr.resistant_susceptible_n AS resistant_susceptible_n, + kdr.resistant_susceptible_percent AS resistant_susceptible_percent, + kdr.resistant_resistant_n AS resistant_resistant_n, + kdr.resistant_resistant_percent AS resistant_resistant_percent, + v995a.vgsc995l_percent AS vgsc995l_percent, + v995a.vgsc995f_percent AS vgsc995f_percent, + v995a.vgsc995s_percent AS vgsc995s_percent, + v995a.vgsc995c_percent AS vgsc995c_percent, + v995a.kdr_percent AS kdr_percent, + v402g.vgsc402v_vgsc402v_n AS vgsc402v_vgsc402v_n, + v402g.vgsc402v_vgsc402v_percent AS vgsc402v_vgsc402v_percent, + v402g.vgsc402v_vgsc402l_n AS vgsc402v_vgsc402l_n, + v402g.vgsc402v_vgsc402l_percent AS vgsc402v_vgsc402l_percent, + v402g.vgsc402l_vgsc402l_n AS vgsc402l_vgsc402l_n, + v402g.vgsc402l_vgsc402l_percent AS vgsc402l_vgsc402l_percent, + v402a.vgsc402v_percent AS vgsc402v_percent, + v402a.vgsc_402l_percent AS vgsc_402l_percent, + v1570g.vgsc1570n_vgsc1570n_n AS vgsc1570n_vgsc1570n_n, + v1570g.vgsc1570n_vgsc1570n_percent AS vgsc1570n_vgsc1570n_percent, + v1570g.vgsc1570n_vgsc1570y_n AS vgsc1570n_vgsc1570y_n, + v1570g.vgsc1570n_1570y_percent AS vgsc1570n_1570y_percent, + v1570g.vgsc1570y_vgsc1570y_n AS vgsc1570y_vgsc1570y_n, + v1570g.vgsc1570y_vgsc1570y_percent AS vgsc1570y_vgsc1570y_percent, + v1570a.vgsc1570n_percent AS vgsc1570n_percent, + v1570a.vgsc1570y_percent AS vgsc1570y_percent, + rms.rdl_method_1 AS rdl_method_1, + rms.rdl_number_of_mosquitoes_tested AS rdl_number_of_mosquitoes_tested, + rms.rdl_generation AS rdl_generation, + rdlg.rdl296c_rdl296c__n AS rdl296c_rdl296c__n, + rdlg.rdl296c_rdl296c_percent AS rdl296c_rdl296c_percent, + rdlg.rdl296c_rdl296g_n AS rdl296c_rdl296g_n, + rdlg.rdl296c_rdl296g_percent AS rdl296c_rdl296g_percent, + rdlg.rdl296g_rdl296g_n AS rdl296g_rdl296g_n, + rdlg.rdl296g_rdl296g_percent AS rdl296g_rdl296g_percent, + rdlg.rdl296c_rdl296s_n AS rdl296c_rdl296s_n, + rdlg.rdl296c_rdl296s_percent AS rdl296c_rdl296s_percent, + rdlg.rdl296s_rdl296s_n AS rdl296s_rdl296s_n, + rdlg.rdl296s_rdl296s_percent AS rdl296s_rdl296s_percent, + rdlg.rdl296g_rdl296s_n AS rdl296g_rdl296s_n, + rdlg.rdl296g_rdl296s_percent AS rdl296g_rdl296s_percent, + rdla.rdl296c_percent AS rdl296c_percent, + rdla.rdl296g_percent AS rdl296g_percent, + rdla.rdl296s_percent AS rdl296s_percent, + ams.ace1_method_1 AS ace1_method_1, + ams.ace1_number_of_mosquitoes_tested AS ace1_number_of_mosquitoes_tested, + ams.ace1_generation AS ace1_generation, + aceg.ace1_280g_ace1_280g_n AS ace1_280g_ace1_280g_n, + aceg.ace1_280g_ace1_280g_percent AS ace1_280g_ace1_280g_percent, + aceg.ace1_280g_ace1_280s_n AS ace1_280g_ace1_280s_n, + aceg.ace1_280g_ace1_280s_percent AS ace1_280g_ace1_280s_percent, + aceg.ace1_280s_ace1_280s_n AS ace1_280s_ace1_280s_n, + aceg.ace1_280s_ace1_280s_percent AS ace1_280s_ace1_280s_percent, + acea.ace1_280g_percent AS ace1_280g_percent, + acea.ace1_280s_percent AS ace1_280s_percent, + gms.gste_method_1 AS gste_method_1, + gms.gste_number_of_mosquitoes_tested AS gste_number_of_mosquitoes_tested, + gms.gste_generation AS gste_generation, + g114g."gste2_114I_gste2_114I_n" AS "gste2_114I_gste2_114I_n", + g114g."gste2_114I_gste2_114I_percent" AS "gste2_114I_gste2_114I_percent", + g114g."gste2_114I_gste2_114t_n" AS "gste2_114I_gste2_114t_n", + g114g."gste2_114I_gste2_114t_percent" AS "gste2_114I_gste2_114t_percent", + g114g.gste2_114t_gste2_114t_n AS gste2_114t_gste2_114t_n, + g114g.gste2_114t_gste2_114t_percent AS gste2_114t_gste2_114t_percent, + g114a."gste2_114I_percent" AS "gste2_114I_percent", + g114a.gste2_114t_percent AS gste2_114t_percent, + g119g.gste2_119l_gste2_119l_n AS gste2_119l_gste2_119l_n, + g119g.gste2_119l_gste2_119l_percent AS gste2_119l_gste2_119l_percent, + g119g.gste2_119l_gste2_119v_n AS gste2_119l_gste2_119v_n, + g119g.gste2_119l_gste2_119v_percent AS gste2_119l_gste2_119v_percent, + g119g.gste2_119v_gste2_119v_n AS gste2_119v_gste2_119v_n, + g119g.gste2_119v_gste2_119v_percent AS gste2_119v_gste2_119v_percent, + g119a.gste2_119l_percent AS gste2_119l_percent, + g119a.gste2_119v_percent AS gste2_119v_percent, + cms.cyp_method_1 AS cyp_method_1, + cms.cyp_number_of_mosquitoes_tested AS cyp_number_of_mosquitoes_tested, + cms.cyp_generation AS cyp_generation, + c4g.cyp4j5_43l_cyp4j5_43l_n AS cyp4j5_43l_cyp4j5_43l_n, + c4g.cyp4j5_43l_cyp4j5_43l_percent AS cyp4j5_43l_cyp4j5_43l_percent, + c4g.cyp4j5_43l_cyp4j5_43f_n AS cyp4j5_43l_cyp4j5_43f_n, + c4g.cyp4j5_43l_cyp4j5_43f_percent AS cyp4j5_43l_cyp4j5_43f_percent, + c4g.cyp4j5_43f_cyp4j5_43f_n AS cyp4j5_43f_cyp4j5_43f_n, + c4g.cyp4j5_43f_cyp4j5_43f_percent AS cyp4j5_43f_cyp4j5_43f_percent, + c4a.cyp4j5_43l_percent AS cyp4j5_43l_percent, + c4a.cyp4j5_43f_percent AS cyp4j5_43f_percent, + c6pg.cyp6p4_236wt_cyp6p4_236wt_n AS cyp6p4_236wt_cyp6p4_236wt_n, + c6pg.cyp6p4_236wt_cyp6p4_236wt_percent AS cyp6p4_236wt_cyp6p4_236wt_percent, + c6pg.cyp6p4_236wt_cyp6p4_236m_n AS cyp6p4_236wt_cyp6p4_236m_n, + c6pg.cyp6p4_236wt_cyp6p4_236m_percent AS cyp6p4_236wt_cyp6p4_236m_percent, + c6pg.cyp6p4_236m_cyp6p4_236m_n AS cyp6p4_236m_cyp6p4_236m_n, + c6pg.cyp6p4_236m_cyp6p4_236m_percent AS cyp6p4_236m_cyp6p4_236m_percent, + c6pa.cyp6p4_236wt_percent AS cyp6p4_236wt_percent, + c6pa.cyp6p4_236m_percent AS cyp6p4_236m_percent, + c6ag.cyp6aap_wt_cyp6aap_wt_n AS cyp6aap_wt_cyp6aap_wt_n, + c6ag.cyp6aap_wt_cyp6aap_wt_percent AS cyp6aap_wt_cyp6aap_wt_percent, + c6ag.cyp6aap_wt_cyp6aap_dup1_n AS cyp6aap_wt_cyp6aap_dup1_n, + c6ag.cyp6aap_wt_cyp6aap_dup1_percent AS cyp6aap_wt_cyp6aap_dup1_percent, + c6ag.cyp6aap_dup1_cyp6aap_dup1_n AS cyp6aap_dup1_cyp6aap_dup1_n, + c6ag.cyp6aap_dup1_cyp6aap_dup1_percent AS cyp6aap_dup1_cyp6aap_dup1_percent, + c6aa.cyp6aap_wt_percent AS cyp6aap_wt_percent, + c6aa.cyp6aap_dup1_percent AS cyp6aap_dup1_percent +FROM occurrence o +INNER JOIN dataset d ON d.id = o."datasetId" AND d.status = 'Approved' +LEFT JOIN reference r ON r.id = o."referenceId" +LEFT JOIN site s ON s.id = o."siteId" +LEFT JOIN recorded_species rs ON rs.id = o."recordedSpeciesId" +LEFT JOIN sample sa ON sa.id = o."sampleId" +LEFT JOIN bionomics b ON b.id = o."bionomicsId" +LEFT JOIN biology bl ON bl.id = b."biologyId" +LEFT JOIN infection inf ON inf.id = b."infectionId" +LEFT JOIN biting_rate br ON br.id = b."bitingRateId" +LEFT JOIN anthropo_zoophagic az ON az.id = b."anthropoZoophagicId" +LEFT JOIN endo_exophagic eg ON eg.id = b."endoExophagicId" +LEFT JOIN biting_activity ba ON ba.id = b."bitingActivityId" +LEFT JOIN endo_exophily ey ON ey.id = b."endoExophilyId" +LEFT JOIN environment env ON env.id = b."environmentId" +LEFT JOIN "Larval_site" ls ON ls.id = b."larvalSiteId" +LEFT JOIN "insecticideResistanceBioassays" ir ON ir.id = o."insecticideResistanceBioassaysId" +LEFT JOIN "genotypicRepresentativeness" gr ON gr.id = ir."genotypicRepresentativenessId" +LEFT JOIN "vgscMethodAndSample" vms ON vms.id = ir."vgscMethodAndSampleId" +LEFT JOIN "vgscGeneytpeFrequencies" vgf ON vgf.id = ir."vgscGeneytpeFrequenciesId" +LEFT JOIN "kdrGenotypeFrequencies" kdr ON kdr.id = ir."kdrGenotypeFrequenciesId" +LEFT JOIN "vgsc995AlleleFrequencies" v995a ON v995a.id = ir."vgsc995AlleleFrequenciesId" +LEFT JOIN "vgsc402GenotypeFrequencies" v402g ON v402g.id = ir."vgsc402GenotypeFrequenciesId" +LEFT JOIN "vgsc402AlleleFrequencies" v402a ON v402a.id = ir."vgsc402AlleleFrequenciesId" +LEFT JOIN "vgsc1570GenotypeFrequencies" v1570g ON v1570g.id = ir."vgsc1570GenotypeFrequenciesId" +LEFT JOIN "vgsc1570AlleleFrequencies" v1570a ON v1570a.id = ir."vgsc1570AlleleFrequenciesId" +LEFT JOIN "rdlMethodAndSample" rms ON rms.id = ir."rdlMethodAndSampleId" +LEFT JOIN "rdl296GenotypeFrequencies" rdlg ON rdlg.id = ir."rdl296GenotypeFrequenciesId" +LEFT JOIN "rdl296AlleleFrequencies" rdla ON rdla.id = ir."rdl296AlleleFrequenciesId" +LEFT JOIN "ace1MethodAndSample" ams ON ams.id = ir."ace1MethodAndSampleId" +LEFT JOIN "ace1GenotypeFrequencies" aceg ON aceg.id = ir."ace1GenotypeFrequenciesId" +LEFT JOIN "ace1AlleleFrequencies" acea ON acea.id = ir."ace1AlleleFrequenciesId" +LEFT JOIN "gsteMethodAndSample" gms ON gms.id = ir."gsteMethodAndSampleId" +LEFT JOIN "gste2_114GenotypeFrequencies" g114g ON g114g.id = ir."gste2114GenotypeFrequenciesId" +LEFT JOIN "gste2_114AlleleFrequencies" g114a ON g114a.id = ir."gste2114AlleleFrequenciesId" +LEFT JOIN "gste2_119GenotypeFrequencies" g119g ON g119g.id = ir."gste2119GenotypeFrequenciesId" +LEFT JOIN "gste2_119AlleleFrequencies" g119a ON g119a.id = ir."gste2119AlleleFrequenciesId" +LEFT JOIN "cytochromesP450_cypMethodAndSample" cms ON cms.id = ir."cytochromesP450CypMethodAndSampleId" +LEFT JOIN "cyp4j5GenotypeFrequencies" c4g ON c4g.id = ir."cyp4j5GenotypeFrequenciesId" +LEFT JOIN "cyp4j5AlleleFrequencies" c4a ON c4a.id = ir."cyp4j5AlleleFrequenciesId" +LEFT JOIN "cyp6p4GenotypeFrequencies" c6pg ON c6pg.id = ir."cyp6p4GenotypeFrequenciesId" +LEFT JOIN "cyp6p4AlleleFrequencies" c6pa ON c6pa.id = ir."cyp6p4AlleleFrequenciesId" +LEFT JOIN "cyp6aapGenotypeFrequencies" c6ag ON c6ag.id = ir."cyp6aapGenotypeFrequenciesId" +LEFT JOIN "cyp6aapAlleleFrequencies" c6aa ON c6aa.id = ir."cyp6aapAlleleFrequenciesId" +WITH NO DATA`); + + // 3. Unique index, needed for REFRESH ... CONCURRENTLY later + await queryRunner.query( + `CREATE UNIQUE INDEX va_export_mv_occurrence_id_uidx ON va_export_mv (occurrence_id)`, + ); + + // 4. First fill. Later refreshes use CONCURRENTLY (DynamicExportServiceV2.refreshExportView). + await queryRunner.query(`REFRESH MATERIALIZED VIEW va_export_mv`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP MATERIALIZED VIEW IF EXISTS va_export_mv`); + } +} \ No newline at end of file diff --git a/src/API/src/db/shared/dynamic-export.service-v2.ts b/src/API/src/db/shared/dynamic-export.service-v2.ts index 378164e31..78c0252c9 100644 --- a/src/API/src/db/shared/dynamic-export.service-v2.ts +++ b/src/API/src/db/shared/dynamic-export.service-v2.ts @@ -13,6 +13,19 @@ import { formatDate } from 'src/utils'; import { DoiService } from '../doi/doi.service'; import { ExportJob } from 'src/exports/export-job.entity'; +/** + * va_export_mv is a materialized view holding the finished export rows + * (one row per occurrence in an Approved dataset, VA template columns in order). + * Set EXPORT_FROM_MV=true in .env to read downloads from it. + */ +const EXPORT_VIEW = 'va_export_mv'; + +/** + * Read at call time, not at import time: .env may be loaded (ConfigModule) + * after this file is imported, so a top-level constant would always be false. + */ +const exportFromViewEnabled = () => process.env.EXPORT_FROM_MV === 'true'; + const AZURE_EXPORTS_DIRECTORY = process.env.AZURE_EXPORTS_DIRECTORY || 'exports'; @@ -96,6 +109,140 @@ export class DynamicExportServiceV2 { private readonly dataSource: DataSource, ) {} + // --------------------------------------------------------------------------- + // Materialized view (va_export_mv) + // --------------------------------------------------------------------------- + + /** + * Updates the view with the latest data. Call this after a dataset is + * approved. CONCURRENTLY means downloads can keep reading while it runs. + */ + async refreshExportView(): Promise { + const started = Date.now(); + await this.dataSource.query( + `REFRESH MATERIALIZED VIEW CONCURRENTLY ${EXPORT_VIEW}`, + ); + this.logger.log(`${EXPORT_VIEW} refreshed in ${Date.now() - started} ms`); + } + + /** How many rows the view has (all of them, or just the given IDs). */ + private async countViewRows(ids: string[]): Promise { + const [{ n }] = ids.length + ? await this.dataSource.query( + `SELECT count(*)::int AS n FROM ${EXPORT_VIEW} WHERE occurrence_id = ANY($1)`, + [ids], + ) + : await this.dataSource.query( + `SELECT count(*)::int AS n FROM ${EXPORT_VIEW}`, + ); + return n; + } + + /** + * Use the view only when the file would come out the same as the old way: + * - no user filters (the view has no filter columns yet), and + * - for an ID list, every ID is in the view (the view only holds Approved + * datasets, but the ID path does not check approval). + */ + private async canUseView(filters: any, ids: string[]): Promise { + if (!exportFromViewEnabled()) { + this.logger.log("EXPORT_FROM_MV is not 'true'; using the join export"); + return false; + } + if (filters && Object.keys(filters).length > 0) { + this.logger.log('Export has filters; using the join export'); + return false; + } + if (ids.length && (await this.countViewRows(ids)) !== ids.length) { + this.logger.warn( + `Some requested occurrences are not in ${EXPORT_VIEW}; using the join export`, + ); + return false; + } + this.logger.log(`Using ${EXPORT_VIEW} for this export`); + return true; + } + + /** + * If the requested IDs are exactly every row in the view, drop the list so each + * page query does not carry tens of thousands of IDs. Only call after canUseView + * has confirmed every requested ID is in the view. + */ + private async idsOrAll(ids: string[]): Promise { + if (!ids.length) return ids; + const unique = new Set(ids).size; + return unique === (await this.countViewRows([])) ? [] : ids; + } + + /** + * Copies rows from the view into the worksheet, 2000 at a time, in + * occurrence_id order. Each page starts after the last id of the previous + * page ("keyset paging"), so late pages are as fast as early ones. + */ + private async writeRowsFromView( + worksheet: ExcelJS.Worksheet, + sortedColumns: RawTemplateFieldMap[], + ids: string[], + yieldAfter: number, + exportJob: ExportJob, + updateProgressCallback?: (jobId: string, progress: number) => void, + ): Promise { + const pageSize = 2000; + const total = Math.max(await this.countViewRows(ids), 1); + let lastId: string | null = null; + let written = 0; + let page = 0; + + while (true) { + const where: string[] = []; + const params: any[] = []; + if (ids.length) { + params.push(ids); + where.push(`occurrence_id = ANY($${params.length})`); + } + if (lastId !== null) { + params.push(lastId); + where.push(`occurrence_id > $${params.length}`); + } + params.push(pageSize); + + const rows: Record[] = await this.dataSource.query( + `SELECT * FROM ${EXPORT_VIEW} + ${where.length ? 'WHERE ' + where.join(' AND ') : ''} + ORDER BY occurrence_id + LIMIT $${params.length}`, + params, + ); + if (!rows.length) break; + + // The view's column names are the template_field names, so copy across directly + for (const r of rows) { + const row: Record = {}; + for (const col of sortedColumns) { + row[col.template_field] = r[col.template_field]; + } + worksheet.addRow(row).commit(); + } + + written += rows.length; + page++; + if (updateProgressCallback && exportJob) { + updateProgressCallback( + exportJob.id, + Math.round((Math.min(written, total) / total) * 85), + ); + } + if (page % yieldAfter === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + + if (rows.length < pageSize) break; + lastId = rows[rows.length - 1].occurrence_id; + } + + this.logger.log(`Exported ${written} rows from ${EXPORT_VIEW}`); + } + /** * Build indexed metadata tree for dynamic relation navigation */ @@ -278,6 +425,29 @@ export class DynamicExportServiceV2 { const approvedIds = occurrenceIds || []; + // New: read the finished rows from the materialized view when possible + if (await this.canUseView(filters, approvedIds)) { + await this.writeRowsFromView( + worksheet, + sortedColumns, + await this.idsOrAll(approvedIds), + yieldAfter, + exportJob, + updateProgressCallback, + ); + await this.makeFiltersAndDOISheet( + workbook, + filters, + generateDoi, + exportJob, + ); + await worksheet.commit(); + await workbook.commit(); + return finalPath; + } + + // Otherwise: the original join export below, unchanged + // If no occurrenceIds provided, fetch all occurrences using filters. // When no IDs are provided, we also filter by dataset.status = 'Approved' // to ensure only data from approved datasets is exported. From 4e1ae240a17f617b07bd7600921387c75549aec1 Mon Sep 17 00:00:00 2001 From: IndianaEunice Date: Tue, 17 Nov 2026 16:17:46 +0300 Subject: [PATCH 2/4] view fix --- src/API/src/db/shared/dynamic-export.service-v2.ts | 12 ++++-------- src/UI/components/map/layers/downloadDataControl.tsx | 9 ++++++++- src/UI/components/map/mapView/MapHUD-v3.tsx | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/API/src/db/shared/dynamic-export.service-v2.ts b/src/API/src/db/shared/dynamic-export.service-v2.ts index 78c0252c9..e09ab570a 100644 --- a/src/API/src/db/shared/dynamic-export.service-v2.ts +++ b/src/API/src/db/shared/dynamic-export.service-v2.ts @@ -138,19 +138,15 @@ export class DynamicExportServiceV2 { return n; } - /** - * Use the view only when the file would come out the same as the old way: - * - no user filters (the view has no filter columns yet), and - * - for an ID list, every ID is in the view (the view only holds Approved - * datasets, but the ID path does not check approval). - */ + + private async canUseView(filters: any, ids: string[]): Promise { if (!exportFromViewEnabled()) { this.logger.log("EXPORT_FROM_MV is not 'true'; using the join export"); return false; } - if (filters && Object.keys(filters).length > 0) { - this.logger.log('Export has filters; using the join export'); + if (!ids.length && filters && Object.keys(filters).length > 0) { + this.logger.log('Export has filters and no occurrence IDs; using the join export'); return false; } if (ids.length && (await this.countViewRows(ids)) !== ids.length) { diff --git a/src/UI/components/map/layers/downloadDataControl.tsx b/src/UI/components/map/layers/downloadDataControl.tsx index 74df88ffb..4ef3fc3bc 100644 --- a/src/UI/components/map/layers/downloadDataControl.tsx +++ b/src/UI/components/map/layers/downloadDataControl.tsx @@ -165,7 +165,14 @@ export const DownloadDataControl = () => { } const ids = (filteredOccurrenceData || []).map((e) => e.id); + // Block the export when the map shows no records + if (!ids.length) { + toast.error(('downloadData.errors.noRecords')); + return; + } const hasEmail = email.trim().length > 0 && isValidEmail(email); + + dispatch( triggerBackgroundExport({ @@ -491,4 +498,4 @@ export const DownloadDataControl = () => { ); -}; +}; \ No newline at end of file diff --git a/src/UI/components/map/mapView/MapHUD-v3.tsx b/src/UI/components/map/mapView/MapHUD-v3.tsx index 3fdca3898..24cd62b78 100644 --- a/src/UI/components/map/mapView/MapHUD-v3.tsx +++ b/src/UI/components/map/mapView/MapHUD-v3.tsx @@ -211,7 +211,7 @@ const MapHUD: React.FC = ({ const oSpecies = String(o.species || '') .toLowerCase() .trim(); - if (!allSelectedSpecies.includes(o.species)) return false; + if (!allSelectedSpecies.includes(oSpecies)) return false; } // 2. Country Filter (Robust with Alternative Names) @@ -306,7 +306,7 @@ const MapHUD: React.FC = ({ return true; // If it passes all checks, keep it! }); - }, [occurrenceData, filters, doiIdSet, doiResolved]); + }, [occurrenceData, filters, doiIdSet, doiResolved,dbCountryData]); const OTHER_LABEL = 'others'; From 58ebd7b3096adc321cde7b8fa1b5affa4ec0b545 Mon Sep 17 00:00:00 2001 From: IndianaEunice Date: Tue, 17 Nov 2026 16:49:40 +0300 Subject: [PATCH 3/4] view query fix --- .../db/shared/dynamic-export.service-v2.ts | 101 +++++------------- src/UI/components/map/mapView/MapHUD-v3.tsx | 2 +- 2 files changed, 28 insertions(+), 75 deletions(-) diff --git a/src/API/src/db/shared/dynamic-export.service-v2.ts b/src/API/src/db/shared/dynamic-export.service-v2.ts index e09ab570a..57e927d0d 100644 --- a/src/API/src/db/shared/dynamic-export.service-v2.ts +++ b/src/API/src/db/shared/dynamic-export.service-v2.ts @@ -13,33 +13,18 @@ import { formatDate } from 'src/utils'; import { DoiService } from '../doi/doi.service'; import { ExportJob } from 'src/exports/export-job.entity'; -/** - * va_export_mv is a materialized view holding the finished export rows - * (one row per occurrence in an Approved dataset, VA template columns in order). - * Set EXPORT_FROM_MV=true in .env to read downloads from it. - */ +/** Materialized view of finished export rows (Approved datasets only). */ const EXPORT_VIEW = 'va_export_mv'; -/** - * Read at call time, not at import time: .env may be loaded (ConfigModule) - * after this file is imported, so a top-level constant would always be false. - */ +/** Read at call time, so .env is loaded first. */ const exportFromViewEnabled = () => process.env.EXPORT_FROM_MV === 'true'; const AZURE_EXPORTS_DIRECTORY = process.env.AZURE_EXPORTS_DIRECTORY || 'exports'; /** - * Non-circular relation tree for the data export. - * - * This is a curated subset of the Occurrence entity's relations that excludes - * back-references (e.g., Bionomics -> Occurrence, Reference -> Bionomics) to - * avoid circular JOINs. Each table is joined exactly once, producing a single - * SQL query per batch via `relationLoadStrategy: 'join'`. - * - * Total: 40 tables, 615 columns — well within PostgreSQL's 1664-column limit. - * - * See docs/export-relation-tree.md for the full tree diagram and rationale. + * Non-circular relation tree (no back-references), so each table joins once. + * See docs/export-relation-tree.md. */ const EXPORT_RELATIONS = { reference: true, @@ -113,10 +98,7 @@ export class DynamicExportServiceV2 { // Materialized view (va_export_mv) // --------------------------------------------------------------------------- - /** - * Updates the view with the latest data. Call this after a dataset is - * approved. CONCURRENTLY means downloads can keep reading while it runs. - */ + /** CONCURRENTLY lets downloads keep reading during the refresh. */ async refreshExportView(): Promise { const started = Date.now(); await this.dataSource.query( @@ -125,7 +107,7 @@ export class DynamicExportServiceV2 { this.logger.log(`${EXPORT_VIEW} refreshed in ${Date.now() - started} ms`); } - /** How many rows the view has (all of them, or just the given IDs). */ + /** Counts all view rows, or only the given IDs. */ private async countViewRows(ids: string[]): Promise { const [{ n }] = ids.length ? await this.dataSource.query( @@ -138,15 +120,20 @@ export class DynamicExportServiceV2 { return n; } - - + /** + * Uses the view only when the output matches the join export. + * Filters need the join, since the view has no filter columns. + * ID lists need every ID in the view. + */ private async canUseView(filters: any, ids: string[]): Promise { if (!exportFromViewEnabled()) { this.logger.log("EXPORT_FROM_MV is not 'true'; using the join export"); return false; } if (!ids.length && filters && Object.keys(filters).length > 0) { - this.logger.log('Export has filters and no occurrence IDs; using the join export'); + this.logger.log( + 'Export has filters and no occurrence IDs; using the join export', + ); return false; } if (ids.length && (await this.countViewRows(ids)) !== ids.length) { @@ -159,22 +146,14 @@ export class DynamicExportServiceV2 { return true; } - /** - * If the requested IDs are exactly every row in the view, drop the list so each - * page query does not carry tens of thousands of IDs. Only call after canUseView - * has confirmed every requested ID is in the view. - */ + /** Returns [] when the IDs cover the whole view, so pages skip the ID filter. */ private async idsOrAll(ids: string[]): Promise { if (!ids.length) return ids; const unique = new Set(ids).size; return unique === (await this.countViewRows([])) ? [] : ids; } - /** - * Copies rows from the view into the worksheet, 2000 at a time, in - * occurrence_id order. Each page starts after the last id of the previous - * page ("keyset paging"), so late pages are as fast as early ones. - */ + /** Copies view rows to the worksheet in pages of 2000, keyed by occurrence_id. */ private async writeRowsFromView( worksheet: ExcelJS.Worksheet, sortedColumns: RawTemplateFieldMap[], @@ -211,7 +190,7 @@ export class DynamicExportServiceV2 { ); if (!rows.length) break; - // The view's column names are the template_field names, so copy across directly + // View columns already use template_field names for (const r of rows) { const row: Record = {}; for (const col of sortedColumns) { @@ -239,9 +218,7 @@ export class DynamicExportServiceV2 { this.logger.log(`Exported ${written} rows from ${EXPORT_VIEW}`); } - /** - * Build indexed metadata tree for dynamic relation navigation - */ + /** Builds a metadata tree of relations for dynamic lookups. */ private buildRelationIndex( repository: Repository, maxDepth = 5, @@ -365,9 +342,7 @@ export class DynamicExportServiceV2 { return row; } - /** - * Main memory-efficient background exporter. Writes directly to disk stream. - */ + /** Streams the export to disk. Uses the view when possible. */ async exportAllToExcelBackground( filters: any, columns: RawTemplateFieldMap[], @@ -392,11 +367,11 @@ export class DynamicExportServiceV2 { targetFilePath || path.join(exportDir, `va_export_${exportJob?.id || Date.now()}.xlsx`); - // 1. Initialize Streaming Workbook Writer directly to local file + // Shared strings and styles stay off to keep memory low const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ filename: finalPath, - useSharedStrings: false, // Prevents holding string dictionary in V8 Heap - useStyles: false, // Disables keeping style objects in memory + useSharedStrings: false, + useStyles: false, }); const worksheet = workbook.addWorksheet('Data'); @@ -412,7 +387,6 @@ export class DynamicExportServiceV2 { const relationIndex = this.buildRelationIndex(this.repository); - // 2. Add columns and flush header row worksheet.columns = sortedColumns.map((col) => ({ header: col.template_field, key: col.template_field, @@ -421,7 +395,6 @@ export class DynamicExportServiceV2 { const approvedIds = occurrenceIds || []; - // New: read the finished rows from the materialized view when possible if (await this.canUseView(filters, approvedIds)) { await this.writeRowsFromView( worksheet, @@ -442,30 +415,20 @@ export class DynamicExportServiceV2 { return finalPath; } - // Otherwise: the original join export below, unchanged - - // If no occurrenceIds provided, fetch all occurrences using filters. - // When no IDs are provided, we also filter by dataset.status = 'Approved' - // to ensure only data from approved datasets is exported. + // With no IDs, only Approved datasets are exported const useAllOccurrences = !approvedIds || approvedIds.length === 0; - // When fetching all occurrences (no explicit IDs), add a filter - // for approved datasets only. This ensures exports don't include - // data from pending or rejected datasets. const exportFilters = useAllOccurrences ? { ...(filters || {}), dataset: { status: 'Approved' } } : filters || {}; - // Get the true total row count for accurate progress reporting. - // When downloading by occurrence IDs, the total is the ID count. - // When downloading by filters, we need a COUNT query so progress - // doesn't jump to 85% after the first batch. + // Total for progress: a COUNT when filtering, the ID count otherwise let total: number; if (useAllOccurrences) { total = await this.repository.count({ where: exportFilters, }); - if (total === 0) total = 1; // avoid division by zero if no rows + if (total === 0) total = 1; } else { total = Math.max(approvedIds.length, 1); } @@ -477,7 +440,6 @@ export class DynamicExportServiceV2 { const skip = page * pageSize; if (useAllOccurrences) { - // Fetch all occurrences with filters and pagination in a single JOIN query entities = await this.repository.find({ where: exportFilters, relations: EXPORT_RELATIONS as any, @@ -488,7 +450,6 @@ export class DynamicExportServiceV2 { }); if (!entities.length) break; } else { - // Use the existing ID-based approach with a single JOIN query const ids = approvedIds.slice(skip, skip + pageSize); if (!ids.length) break; @@ -501,10 +462,9 @@ export class DynamicExportServiceV2 { if (!entities.length) break; } - // Map each entity to row and commit immediately to disk for (const entity of entities) { const row = this.mapEntityToRow(entity, sortedColumns, relationIndex); - worksheet.addRow(row).commit(); // Flush row memory immediately + worksheet.addRow(row).commit(); } page++; @@ -515,14 +475,12 @@ export class DynamicExportServiceV2 { updateProgressCallback(exportJob.id, progress); } - // yield every x pages - // Potentially gives other HTTP requests and health checks an immediate turn on the event loop + // Lets other requests run during long exports if (page % yieldAfter === 0) { await new Promise((resolve) => setImmediate(resolve)); } } - // Add Filters and DOI Worksheet await this.makeFiltersAndDOISheet( workbook, filters, @@ -530,7 +488,6 @@ export class DynamicExportServiceV2 { exportJob, ); - // Finalize file write await worksheet.commit(); await workbook.commit(); @@ -553,12 +510,10 @@ export class DynamicExportServiceV2 { this.logger.debug(`Processing ${Object.keys(filters).length} filters`); Object.keys(filters).forEach((element) => { const val = filters[element]; - // Convert complex values to readable strings let displayValue = val; if (Array.isArray(val)) { displayValue = val.join(', '); } else if (typeof val === 'object' && val !== null) { - // Handle nested timeRange object if it wasn't sanitized if (element === 'timeRange' && val.start !== undefined) { const startDate = val.start ? new Date(val.start).toISOString().split('T')[0] @@ -574,7 +529,6 @@ export class DynamicExportServiceV2 { typeof val === 'number' && (element === 'startTimestamp' || element === 'endTimestamp') ) { - // Convert Unix timestamp to human-readable date displayValue = new Date(val).toISOString().split('T')[0]; } this.logger.debug(`Adding filter row: ${element}=${displayValue}`); @@ -610,7 +564,6 @@ export class DynamicExportServiceV2 { doi.title = 'Data Download - ' + formatDate(new Date()); doi.approval_status = ApprovalStatus.PENDING; - // Create a human-readable version of filters for description const readableFilters: any = {}; Object.keys(filters).forEach((key) => { const val = filters[key]; diff --git a/src/UI/components/map/mapView/MapHUD-v3.tsx b/src/UI/components/map/mapView/MapHUD-v3.tsx index 24cd62b78..b8adbd534 100644 --- a/src/UI/components/map/mapView/MapHUD-v3.tsx +++ b/src/UI/components/map/mapView/MapHUD-v3.tsx @@ -306,7 +306,7 @@ const MapHUD: React.FC = ({ return true; // If it passes all checks, keep it! }); - }, [occurrenceData, filters, doiIdSet, doiResolved,dbCountryData]); + }, [occurrenceData, filters, doiIdSet, doiResolved, dbCountryData]); const OTHER_LABEL = 'others'; From 3aa658f32cbe1dbb69b8a78458da634fa3711317 Mon Sep 17 00:00:00 2001 From: IndianaEunice Date: Tue, 17 Nov 2026 16:56:49 +0300 Subject: [PATCH 4/4] view fix --- src/UI/components/map/layers/downloadDataControl.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/UI/components/map/layers/downloadDataControl.tsx b/src/UI/components/map/layers/downloadDataControl.tsx index 4ef3fc3bc..ad15037b6 100644 --- a/src/UI/components/map/layers/downloadDataControl.tsx +++ b/src/UI/components/map/layers/downloadDataControl.tsx @@ -167,12 +167,10 @@ export const DownloadDataControl = () => { const ids = (filteredOccurrenceData || []).map((e) => e.id); // Block the export when the map shows no records if (!ids.length) { - toast.error(('downloadData.errors.noRecords')); + toast.error(t('downloadData.errors.noRecords')); return; } const hasEmail = email.trim().length > 0 && isValidEmail(email); - - dispatch( triggerBackgroundExport({ @@ -498,4 +496,4 @@ export const DownloadDataControl = () => { ); -}; \ No newline at end of file +};