Config changes us hud income limits

scripts/us_hud/income/golden_data/golden_summary_report.csv

@@ -0,0 +1,17 @@
+"MinDate","ScalingFactors","MeasurementMethods","StatVar","observationPeriods","Units","NumPlaces"
+"2006","[]","[]","Median_Income_Household_With1Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With5Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With2Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With6Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With8Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With7Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With3Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With1Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With4Person_150Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With4Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With3Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With5Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With2Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With8Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With7Person_80Percentile","[P1Y]","[USDollar]","5170"
+"2006","[]","[]","Median_Income_Household_With6Person_80Percentile","[P1Y]","[USDollar]","5170"

scripts/us_hud/income/manifest.json

@@ -10,6 +10,7 @@
             "scripts": [
                 "process.py"
             ],
+
             "import_inputs": [
                 {
                     "template_mcf": "hud.tmcf",
@@ -19,7 +20,8 @@
             "source_files": [
                 "input_files/*"
             ],
-            "cron_schedule": "15 23 * * 7"
+            "cron_schedule": "15 23 * * 7",
+            "validation_config_file": "validation_config.json"
         }
     ]
 }

scripts/us_hud/income/process.py

@@ -146,15 +146,28 @@ def process_all():
 
     if FLAGS.mode == "" or FLAGS.mode == "download":
         logging.info("Starting download phase...")
-        for year in range(2006, today.year):
+        for year in range(2006, today.year + 1):
             url = get_url(year)
             if url:
                 filename = f"Section8-FY{year}.xlsx" if year > 2016 else f"Section8-FY{year}.xls"
+                if year == today.year:
+                    try:
+                        headers = {
+                            'User-Agent':
+                                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+                        }
+                        resp = requests.get(url, headers=headers, timeout=10, stream=True)
+                        if resp.status_code != 200:
+                            logging.warning(f"HUD income limits for {year} are not yet available. Skipping.")
+                            continue
+                    except Exception as e:
+                        logging.warning(f"Could not check availability for {year}: {e}. Skipping.")
+                        continue
                 download_file(url, filename, input_folder)
 
     if FLAGS.mode == "" or FLAGS.mode == "process":
         logging.info("Starting processing phase...")
-        for year in range(2006, today.year):
+        for year in range(2006, today.year + 1):
             if not os.path.exists(
                     os.path.join(
                         input_folder, f"Section8-FY{year}.xlsx"

scripts/us_hud/income/process_test.py

@@ -42,6 +42,23 @@ def test_get_url(self):
         )
         year = 1997
         self.assertEqual(process.get_url(year), '')
+
+        # Test the current year URL generation, as process.py now supports processing up to the current year
+        import datetime
+        current_year = datetime.date.today().year
+        suffix = str(current_year)[-2:]
+        self.assertEqual(
+            process.get_url(current_year),
+            f'https://www.huduser.gov/portal/datasets/il/il{suffix}/Section8-FY{suffix}.xlsx'
+        )
+
+        # Test the next year (today.year + 1) URL generation, matching the upper limit of process.py's loop range
+        next_year = current_year + 1
+        next_suffix = str(next_year)[-2:]
+        self.assertEqual(
+            process.get_url(next_year),
+            f'https://www.huduser.gov/portal/datasets/il/il{next_suffix}/Section8-FY{next_suffix}.xlsx'
+        )
 
     def test_process_with_dynamic_csv(self):
         matches = {'dcs:geoId/02110': 'dcs:geoId/0236400'}

scripts/us_hud/income/validation_config.json

@@ -0,0 +1,20 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Strictly enforce historical deletion average threshold of 0.1%",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        }
+    ]
+}