From 147c64376f12000b8f51abe4c9a99b2c08060441 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 7 Jul 2026 08:52:34 +0000 Subject: [PATCH 1/4] testing --- .../golden_data/golden_summary_report.csv | 7 +++++++ .../education_attainment/manifest.json | 3 ++- .../validation_config.json | 20 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv create mode 100644 scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..c8b2f0cfa8 --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/golden_data/golden_summary_report.csv @@ -0,0 +1,7 @@ +"StatVar","MeasurementMethods","ScalingFactors","NumPlaces","MinDate","Units","observationPeriods" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_AsAFractionOfCount_Person_25To64Years","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_Female_AsAFractionOfCount_Person_25To64Years_Female","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_Male_AsAFractionOfCount_Person_25To64Years_Male","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_Male_AsAFractionOfCount_Person_25To64Years_Male","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_UpperSecondaryEducationOrHigher_Female_AsAFractionOfCount_Person_25To64Years_Female","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" +"Count_Person_25To64Years_TertiaryEducation_AsAFractionOfCount_Person_25To64Years","[EurostatRegionalStatistics]","[100]","512","2000","[Percent]","[]" diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json index a140431d7b..e44fb04003 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/manifest.json @@ -20,7 +20,8 @@ "edat_lfse_04.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "30 02 * * 2" + "cron_schedule": "30 02 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json new file mode 100644 index 0000000000..ef5bb8f3dc --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.05 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +} From 34f11bbf9815c1e299347e90a111868bb621b513 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 7 Jul 2026 12:42:00 +0000 Subject: [PATCH 2/4] testing --- .../education_attainment/validation_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json index ef5bb8f3dc..791c40c55e 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_attainment/validation_config.json @@ -6,7 +6,7 @@ "description": "Checks that the percentage of deleted points is within the threshold.", "validator": "DELETED_RECORDS_PERCENT", "params": { - "threshold": 0.05 + "threshold": 0.1 } }, { From 59fae70da2ab9af4e5137ab467ed30e04fd4760b Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 8 Jul 2026 06:41:08 +0000 Subject: [PATCH 3/4] added golden eurostatdata_education_enrollment --- .../golden_data/golden_summary_report.csv | 4 ++++ .../education_enrollment/manifest.json | 3 ++- .../validation_config.json | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv create mode 100644 scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..97a32c014a --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/golden_data/golden_summary_report.csv @@ -0,0 +1,4 @@ +"Units","MeasurementMethods","ScalingFactors","observationPeriods","NumPlaces","StatVar","MinDate" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","512","Count_Person_25To64Years_EnrolledInEducationOrTraining_AsAFractionOfCount_Person_25To64Years","2000" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","509","Count_Person_25To64Years_EnrolledInEducationOrTraining_Male_AsAFractionOfCount_Person_25To64Years_Male","2000" +"[Percent]","[EurostatRegionalStatistics]","[100]","[]","511","Count_Person_25To64Years_EnrolledInEducationOrTraining_Female_AsAFractionOfCount_Person_25To64Years_Female","2000" diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json index 60ac5e00a8..8000fbf0c7 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/manifest.json @@ -20,7 +20,8 @@ "trng_lfse_04.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "00 03 * * 2" + "cron_schedule": "00 03 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json new file mode 100644 index 0000000000..e3f52fbeeb --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/education_enrollment/validation_config.json @@ -0,0 +1,21 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +} + From 8bf3237063165b7eda849e7769463f1ef366de9e Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 8 Jul 2026 06:43:30 +0000 Subject: [PATCH 4/4] added golden eurostatdata_birth_death_migration --- .../golden_data/golden_summary_report.csv | 8 ++++++++ .../birth_death_migration/manifest.json | 3 ++- .../validation_config.json | 20 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv create mode 100644 scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..4c5595abf6 --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/golden_data/golden_summary_report.csv @@ -0,0 +1,8 @@ +"Units","StatVar","ScalingFactors","observationPeriods","NumPlaces","MeasurementMethods","MinDate" +"[]","Count_Death_AsAFractionOfCount_Person","[]","[]","2122","[EurostatRegionalStatistics]","2000" +"[]","GrowthRate_Count_Person","[]","[]","2122","[EurostatRegionalStatistics]","2000" +"[]","Count_Death","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_Person","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_BirthEvent","[]","[]","2124","[EurostatRegionalStatistics]","2000" +"[]","IncrementalCount_Person","[]","[]","2125","[EurostatRegionalStatistics]","2000" +"[]","Count_BirthEvent_AsAFractionOfCount_Person","[]","[]","2121","[EurostatRegionalStatistics]","2000" diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json index b5820425f0..2c43154003 100644 --- a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json @@ -20,7 +20,8 @@ "demo_r_gind3.tsv.gz", "input_files/input_file.tsv" ], - "cron_schedule": "00 02 * * 2" + "cron_schedule": "00 02 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json new file mode 100644 index 0000000000..791c40c55e --- /dev/null +++ b/scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/validation_config.json @@ -0,0 +1,20 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + } + ] +}