USGS Fix & Other Updates

README.md

@@ -38,8 +38,9 @@ Data comes from the following sources. We are continuously adding new sources as
   - Available data: `water levels`
 - [Pecos Valley Artesian Conservancy District (PVACD)](https://st2.newmexicowaterdata.org/FROST-Server/v1.1/Locations?$filter=properties/agency%20eq%20%27PVACD%27)
   - Available data: `water levels`
-- [USGS (NWIS)](https://waterdata.usgs.gov/nwis)
+- [USGS (NWIS)](https://api.waterdata.usgs.gov/docs/)
   - Available data: `water levels`
+  - **IMPORTANT:** The USGS now uses API keys. To prevent yourself from hitting the rate limit please [acquire an API key](https://api.waterdata.usgs.gov/signup/), save it, and provide it via the `--usgs-api-key` flag when gathering water level data from the USGS.
 - [Water Quality Portal (WQP)](https://www.waterqualitydata.us/)
   - Available data: `water levels`, `water quality`
 
@@ -189,6 +190,14 @@ The Data Integration Engine enables the user to obtain groundwater level and gro
 - `--no-pvacd` to exclude Pecos Valley Artesian Convservancy District (PVACD) data
 - `--no-wqp` to exclude Water Quality Portal (WQP) data
 
+### USGS API Keys
+
+The USGS now uses [API keys](https://api.waterdata.usgs.gov/signup/) to increase the query rate limit to their APIs. If you intend to include USGS water level data in your output please acquire an API key, save it somewhere, and provide it via the `--usgs-api-key` flag. For example:
+
+```
+die weave waterlevels --output-type timeseries_unified --usgs-api-key FAKE_API_KEY
+```
+
 ### Geographic Filters [In Development]
 
 The following flags can be used to geographically filter data:

backend/config.py

@@ -281,8 +281,19 @@ def get_config_and_false_agencies(self):
                 "nwis",
                 "pvacd",
             ]
+        elif self.parameter in [BICARBONATE]:
+            config_agencies = ["nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]
+            false_agencies = [
+                "bor",
+                "bernco",
+                "cabq",
+                "ebid",
+                "nmose_roswell",
+                "nmose_pod",
+                "nwis",
+                "pvacd",
+            ]
         elif self.parameter in [
-            BICARBONATE,
             CALCIUM,
             CHLORIDE,
             FLUORIDE,

backend/connectors/bor/source.py

@@ -57,7 +57,7 @@ def get_records(self):
         # locationTypeId 10 is for wells
         url = "https://data.usbr.gov/rise/api/location"
         params = {"stateId": "NM", "locationTypeId": 10}
-        return self._execute_json_request(url, params)
+        return self._execute_json_request(url, params, tag="data")
 
 
 def parse_dt(dt):
@@ -119,13 +119,14 @@ def get_records(self, site_record):
         code = get_analyte_search_param(self.config.parameter, BOR_ANALYTE_MAPPING)
 
         catalog_record_data = self._execute_json_request(
-            f"https://data.usbr.gov{site_record.catalogRecords[0]['id']}"
+            f"https://data.usbr.gov{site_record.catalogRecords[0]['id']}",
+            tag="data"
         )
         catalog_items = catalog_record_data["relationships"]["catalogItems"]["data"]
 
         for i, item in enumerate(self._reorder_catalog_items(catalog_items)):
 
-            data = self._execute_json_request(f'https://data.usbr.gov{item["id"]}')
+            data = self._execute_json_request(f'https://data.usbr.gov{item["id"]}', tag="data")
             if not data:
                 continue
 
@@ -142,6 +143,7 @@ def get_records(self, site_record):
                 return self._execute_json_request(
                     "https://data.usbr.gov/rise/api/result",
                     params={"itemId": data["attributes"]["_id"]},
+                    tag="data"
                 )
 
 

backend/connectors/isc_seven_rivers/source.py

@@ -85,6 +85,7 @@ def health(self):
     def get_records(self):
         return self._execute_json_request(
             _make_url("getMonitoringPoints.ashx"),
+            tag="data",
         )
 
 
@@ -100,7 +101,7 @@ def _get_analyte_id_and_name(self, analyte):
         """ """
         if self._analyte_ids is None:
 
-            resp = self._execute_json_request(_make_url("getAnalytes.ashx"))
+            resp = self._execute_json_request(_make_url("getAnalytes.ashx"), tag="data")
             if resp:
                 self._analyte_ids = {r["name"]: r["id"] for r in resp}
 
@@ -164,7 +165,7 @@ def get_records(self, site_record):
                 self._source_parameter_name = analyte_id_and_name["name"]
 
             return self._execute_json_request(
-                _make_url("getReadings.ashx"), params=params
+                _make_url("getReadings.ashx"), params=params, tag="data"
             )
 
 
@@ -184,6 +185,7 @@ def get_records(self, site_record):
         return self._execute_json_request(
             _make_url("getWaterLevels.ashx"),
             params=params,
+            tag="data",
         )
 
     def _clean_records(self, records):