From 168e0053681bf7ee922b611eaf165a70cc362553 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 18 Sep 2025 07:20:30 -0600 Subject: [PATCH 01/47] feat: enhance logging and performance tracking in transfer scripts --- transfers/contact_transfer.py | 8 ++--- transfers/link_ids_transfer.py | 7 ++-- transfers/transfer.py | 61 +++++++++++++++++++--------------- transfers/util.py | 30 +++++++++++++---- 4 files changed, 66 insertions(+), 40 deletions(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 053f90ae3..7d2234f8b 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -58,7 +58,7 @@ def transfer_contacts(session): odf = filter_to_valid_point_ids(session, odf) for i, row in odf.iterrows(): thing = session.query(Thing).where(Thing.name == row.PointID).first() - print(f"Processing PointID: {i} {row.PointID}") + logger.info(f"Processing PointID: {i} {row.PointID}") if thing is None: logger.warning( f"Thing with PointID {row.PointID} not found. Skipping owner." @@ -71,7 +71,7 @@ def transfer_contacts(session): session.flush() logger.info(f"added first contact for PointID {row.PointID}") except Exception as e: - logger.warning( + logger.critical( f"Skipping first contact for PointID {row.PointID} due to validation error: {e}" ) from pprint import pprint @@ -83,9 +83,9 @@ def transfer_contacts(session): add_second_contact(session, row, thing) session.commit() session.flush() - print(f"added second contact for PointID {row.PointID}") + logger.info(f"added second contact for PointID {row.PointID}") except Exception as e: - print( + logger.critical( f"Skipping second contact for PointID {row.PointID} due to validation error: {e}" ) session.rollback() diff --git a/transfers/link_ids_transfer.py b/transfers/link_ids_transfer.py index 4748a7e46..6eca741d1 100644 --- a/transfers/link_ids_transfer.py +++ b/transfers/link_ids_transfer.py @@ -98,7 +98,7 @@ def add_link_alternate_site_id(session, row, thing): link_id.alternate_organization = extract_organization(str(row.AlternateSiteID)) - logger.info(f"adding link id: {link_id}") + logger.info(f"adding link id: {row.PointID}") session.add(link_id) @@ -153,7 +153,6 @@ def transfer_link_ids(session, site_type="GW"): ldf = read_csv("Location") ldf = ldf[ldf["SiteType"] == site_type] ldf = ldf[ldf["Easting"].notna() & ldf["Northing"].notna()] - # ldf = ldf[ldf["AlternateSiteID"].notna()] ldf = replace_nans(ldf) ldf = filter_to_valid_point_ids(session, ldf) @@ -166,8 +165,8 @@ def transfer_link_ids(session, site_type="GW"): ) continue logger.info( - f"Processing PointID: {row.PointID}, Thing ID: {thing.id}, a={row.AlternateSiteID}, " - f"b={row.AlternateSiteID2}" + f"Processing PointID: {row.PointID}, Thing ID: {thing.id}, AlternateSiteID={row.AlternateSiteID}, " + f"AlternateSiteID2={row.AlternateSiteID2}" ) add_link_alternate_site_id(session, row, thing) # add_link_site_id(session, row, thing) diff --git a/transfers/transfer.py b/transfers/transfer.py index 5d5be3cbc..7151af852 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -16,6 +16,7 @@ import time from dotenv import load_dotenv +from shapely.linear import line_merge load_dotenv() @@ -36,27 +37,34 @@ transfer_ephemeral_stream, transfer_met, ) -from transfers.util import logger +from transfers.util import logger, timeit, timeit_direct def erase_and_initalize(session: Session) -> None: logger.info("Erasing existing data and initializing lexicon and sensors") - starttime = time.time() - Base.metadata.drop_all(session.bind) - Base.metadata.create_all(session.bind) - elapsed_time = time.time() - starttime - logger.info(f"Done erasing existing data. {elapsed_time:0.2f}s") + erase(session) + lexicon() + sensor(session) - logger.info("Initializing lexicon and sensors") - starttime = time.time() - init_lexicon() - elapsed_time = time.time() - starttime - logger.info(f"Done initializing lexicon. {elapsed_time:0.2f}s") - starttime = time.time() +@timeit +def sensor(session: Session): + logger.info("Initializing sensors") init_sensor(session) - elapsed_time = time.time() - starttime - logger.info(f"Done initializing sensors. {elapsed_time:0.2f}s") + + +@timeit +def lexicon(): + logger.info("Initializing lexicon") + init_lexicon() + + +@timeit +def erase(session: Session): + logger.info("Erasing existing data") + Base.metadata.drop_all(session.bind) + logger.info("Recreating tables") + Base.metadata.create_all(session.bind) def message(msg, pad=10, new_line_at_top=True): @@ -66,6 +74,7 @@ def message(msg, pad=10, new_line_at_top=True): logger.info(f"{pad} {msg} {pad}") +@timeit def main_transfer(): message("STARTING TRANSFER", new_line_at_top=False) @@ -97,44 +106,44 @@ def main_transfer(): cleanup_wells_flag = True - limit = 15 + limit = 100 with session_ctx() as sess: if init: erase_and_initalize(sess) if init or transfer_well_flag: message("TRANSFERRING WELLS") - transfer_wells(sess, limit=limit) - transfer_wellscreens(sess) + timeit_direct(transfer_wells, sess, limit=limit) + timeit_direct(transfer_wellscreens, sess) # if init or transfer_spring_flag: message("TRANSFERRING SPRINGS") - transfer_springs(sess, limit) + timeit_direct(transfer_springs, sess, limit=limit) if init or transfer_perennial_stream_flag: message("TRANSFERRING PERENNIAL STREAMS") - transfer_perennial_stream(sess, limit) + timeit_direct(transfer_perennial_stream, sess, limit=limit) if init or transfer_ephemeral_stream_flag: message("TRANSFERRING EPHEMERAL STREAMS") - transfer_ephemeral_stream(sess, limit) + timeit_direct(transfer_ephemeral_stream, sess, limit=limit) if init or transfer_met_flag: message("TRANSFERRING METEOROLOGICAL") - transfer_met(sess, limit) + timeit_direct(transfer_met, sess, limit) if init or transfer_contacts_flag: message("TRANSFERRING CONTACTS") - transfer_contacts(sess) + timeit_direct(transfer_contacts, sess) if init or transfer_waterlevels_flag: message("TRANSFERRING WATER LEVELS") - transfer_water_levels(sess) + timeit_direct(transfer_water_levels, sess) if init or transfer_link_ids_flag: message("TRANSFERRING LINK IDS") - transfer_link_ids(sess) - transfer_link_ids_welldata(sess) + timeit_direct(transfer_link_ids, sess) + timeit_direct(transfer_link_ids_welldata, sess) # if init or transfer_assets_flag: # message("TRANSFERRING ASSETS") @@ -142,7 +151,7 @@ def main_transfer(): if init or transfer_groups_flag: message("TRANSFERRING GROUPS") - transfer_groups(sess) + timeit_direct(transfer_groups, sess) # if init or cleanup_wells_flag: # cleanup_wells(sess) diff --git a/transfers/util.py b/transfers/util.py index 327c55391..ec4a92557 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -156,9 +156,12 @@ def make_location(row: pd.Series) -> Location: point, source_srid=SRID_UTM_ZONE_13N, target_srid=SRID_WGS84 ) - state = get_state_from_point(transformed_point.x, transformed_point.y) - county = get_county_from_point(transformed_point.x, transformed_point.y) - quad_name = get_quad_name_from_point(transformed_point.x, transformed_point.y) + # since this is such a time consuming operation, I do not want to run it during this step + # cleanup_wells was added for this reason + + # state = get_state_from_point(transformed_point.x, transformed_point.y) + # county = get_county_from_point(transformed_point.x, transformed_point.y) + # quad_name = get_quad_name_from_point(transformed_point.x, transformed_point.y) z = row.Altitude if z: @@ -288,9 +291,10 @@ def make_location(row: pd.Series) -> Location: coordinate_method=coordinate_method, nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, - state=state, - county=county, - quad_name=quad_name, + # these values will be populated in cleanup_wells + # state=state, + # county=county, + # quad_name=quad_name, ) return location @@ -343,6 +347,20 @@ def make_lu_to_lexicon_mapper(): lu_to_lexicon_map = make_lu_to_lexicon_mapper() +def timeit_direct(func, *args, **kwargs): + start = datetime.now() + result = func(*args, **kwargs) + end = datetime.now() + logger.info(f"{func.__name__} took {(end - start).total_seconds()} seconds") + return result + + +def timeit(func): + def wrapper(*args, **kwargs): + return timeit_direct(func, *args, **kwargs) + return wrapper + + if __name__ == "__main__": print(lu_to_lexicon_map) From 50ef63d2e7c9a237b348d981b09f54d294025eb3 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Thu, 18 Sep 2025 13:20:50 +0000 Subject: [PATCH 02/47] Formatting changes --- transfers/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transfers/util.py b/transfers/util.py index ec4a92557..9a9fde085 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -358,6 +358,7 @@ def timeit_direct(func, *args, **kwargs): def timeit(func): def wrapper(*args, **kwargs): return timeit_direct(func, *args, **kwargs) + return wrapper From 6ecbf9a9369a580c75adb81c3e5eecbff542b7ca Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 22 Sep 2025 12:58:07 -0600 Subject: [PATCH 03/47] feat: enhance date handling and error logging in transfer scripts --- core/initializers.py | 2 +- services/lexicon_helper.py | 4 ++-- transfers/util.py | 2 +- transfers/waterlevels_transfer.py | 10 ++++++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/core/initializers.py b/core/initializers.py index 0533d9587..d3ffd9859 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -50,7 +50,7 @@ def init_hypertables(): # session.close() -def init_lexicon(path=None): +def init_lexicon(path: str=None) -> None: if path is None: path = Path(__file__).parent / "lexicon.json" diff --git a/services/lexicon_helper.py b/services/lexicon_helper.py index c4a98454b..7c6c9d49b 100644 --- a/services/lexicon_helper.py +++ b/services/lexicon_helper.py @@ -58,8 +58,8 @@ def add_lexicon_term( ) audit_add(user, category) session.add(category) - session.commit() - session.flush() + # session.commit() + # session.flush() db_categories.append(category) diff --git a/transfers/util.py b/transfers/util.py index 9a9fde085..5b87b7d25 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -280,7 +280,7 @@ def make_location(row: pd.Series) -> Location: location = Location( nma_pk_location=row.LocationId, # name=row.PointID, - point=point.wkt, + point=transformed_point.wkt, elevation=z, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 53eff8d0c..563372f6a 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -54,8 +54,14 @@ def transfer_water_levels(session): else: dt_measured = f"{row.DateMeasured} 12:00:00 AM" - dt = datetime.strptime(dt_measured, "%Y-%m-%d %I:%M:%S %p") - dt_utc = convert_mt_to_utc(dt) + try: + dt = datetime.strptime(dt_measured, "%Y-%m-%d %I:%M:%S %p") + dt_utc = convert_mt_to_utc(dt) + except ValueError as e: + logger.warning( + f"Skipping row {row.Index} due to invalid date/time: {e}" + ) + continue thing = session.query(Thing).where(Thing.name == row.PointID).first() if thing is None: From e19d3602d92bff492316fb579ffe69d01c37c8a9 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Mon, 22 Sep 2025 21:17:18 +0000 Subject: [PATCH 04/47] Formatting changes --- core/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/initializers.py b/core/initializers.py index d3ffd9859..33ad06d73 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -50,7 +50,7 @@ def init_hypertables(): # session.close() -def init_lexicon(path: str=None) -> None: +def init_lexicon(path: str = None) -> None: if path is None: path = Path(__file__).parent / "lexicon.json" From 8aff1a4fdda29c90fd2b1723e76527067b40dad7 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 08:04:10 -0600 Subject: [PATCH 05/47] feat: refactor transfer functions and add main_transfer endpoint --- transfers/entrypoint.py | 8 ++- transfers/transfer.py | 118 ++++++++++++++-------------------------- transfers/util.py | 4 +- 3 files changed, 50 insertions(+), 80 deletions(-) diff --git a/transfers/entrypoint.py b/transfers/entrypoint.py index 082fde659..77d9d99ac 100644 --- a/transfers/entrypoint.py +++ b/transfers/entrypoint.py @@ -27,7 +27,7 @@ transfer_perennial_stream, transfer_springs, ) -from transfers.transfer import message +from transfers.transfer import message, main_transfer, transfer_all from transfers.waterlevels_transfer import transfer_water_levels from transfers.well_transfer import transfer_wells, cleanup_wells @@ -104,6 +104,12 @@ async def _cleanup_wells(session: session_dependency): cleanup_wells(session) +@app.post("/main_transfer") +async def _(session: session_dependency): + message("TRANSFERRING ALL") + transfer_all(session) + + @app.get("/health") async def health(): return {"status": "ok"} diff --git a/transfers/transfer.py b/transfers/transfer.py index 6937487ba..9515a86e5 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -73,88 +73,52 @@ def message(msg, pad=10, new_line_at_top=True): logger.info(f"{pad} {msg} {pad}") -@timeit -def main_transfer(): +def transfer_all(sess, limit=100): message("STARTING TRANSFER", new_line_at_top=False) + erase_and_initalize(sess) + + message("TRANSFERRING WELLS") + timeit_direct(transfer_wells, sess, limit=limit) + timeit_direct(transfer_wellscreens, sess) + + message("TRANSFERRING SPRINGS") + timeit_direct(transfer_springs, sess, limit=limit) + + message("TRANSFERRING PERENNIAL STREAMS") + timeit_direct(transfer_perennial_stream, sess, limit=limit) + + message("TRANSFERRING EPHEMERAL STREAMS") + timeit_direct(transfer_ephemeral_stream, sess, limit=limit) + + message("TRANSFERRING METEOROLOGICAL") + timeit_direct(transfer_met, sess, limit) + + message("TRANSFERRING CONTACTS") + timeit_direct(transfer_contacts, sess) - init = True - - transfer_well_flag = False - transfer_spring_flag = False - transfer_perennial_stream_flag = False - transfer_ephemeral_stream_flag = False - transfer_met_flag = False - transfer_contacts_flag = False - transfer_waterlevels_flag = False - transfer_link_ids_flag = False - transfer_assets_flag = False - transfer_groups_flag = False - - cleanup_wells_flag = False - - transfer_well_flag = True - transfer_spring_flag = True - transfer_perennial_stream_flag = True - transfer_ephemeral_stream_flag = True - transfer_met_flag = True - transfer_contacts_flag = True - transfer_waterlevels_flag = True - transfer_link_ids_flag = True - transfer_assets_flag = True - transfer_groups_flag = True - - cleanup_wells_flag = True + message("TRANSFERRING WATER LEVELS") + timeit_direct(transfer_water_levels, sess) + + message("TRANSFERRING LINK IDS") + timeit_direct(transfer_link_ids, sess) + timeit_direct(transfer_link_ids_welldata, sess) + + # if init or transfer_assets_flag: + # message("TRANSFERRING ASSETS") + # transfer_assets_testing(sess) + + message("TRANSFERRING GROUPS") + timeit_direct(transfer_groups, sess) + + # if init or cleanup_wells_flag: + # cleanup_wells(sess) + +@timeit +def main_transfer(): limit = 100 with session_ctx() as sess: - if init: - erase_and_initalize(sess) - - if init or transfer_well_flag: - message("TRANSFERRING WELLS") - timeit_direct(transfer_wells, sess, limit=limit) - timeit_direct(transfer_wellscreens, sess) - # - if init or transfer_spring_flag: - message("TRANSFERRING SPRINGS") - timeit_direct(transfer_springs, sess, limit=limit) - - if init or transfer_perennial_stream_flag: - message("TRANSFERRING PERENNIAL STREAMS") - timeit_direct(transfer_perennial_stream, sess, limit=limit) - - if init or transfer_ephemeral_stream_flag: - message("TRANSFERRING EPHEMERAL STREAMS") - timeit_direct(transfer_ephemeral_stream, sess, limit=limit) - - if init or transfer_met_flag: - message("TRANSFERRING METEOROLOGICAL") - timeit_direct(transfer_met, sess, limit) - - if init or transfer_contacts_flag: - message("TRANSFERRING CONTACTS") - timeit_direct(transfer_contacts, sess) - - if init or transfer_waterlevels_flag: - message("TRANSFERRING WATER LEVELS") - timeit_direct(transfer_water_levels, sess) - - if init or transfer_link_ids_flag: - message("TRANSFERRING LINK IDS") - timeit_direct(transfer_link_ids, sess) - timeit_direct(transfer_link_ids_welldata, sess) - - # if init or transfer_assets_flag: - # message("TRANSFERRING ASSETS") - # transfer_assets_testing(sess) - - if init or transfer_groups_flag: - message("TRANSFERRING GROUPS") - timeit_direct(transfer_groups, sess) - - # if init or cleanup_wells_flag: - # cleanup_wells(sess) - + transfer_all(sess, limit=limit) if __name__ == "__main__": main_transfer() diff --git a/transfers/util.py b/transfers/util.py index 5b87b7d25..9bc55c630 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -51,8 +51,8 @@ def write(self, buf): def flush(self): pass - -log_filename = f"transfers/transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" +# todo: setup of logging should be moved to function +log_filename = f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" logging.basicConfig( From 1fcf40fb0a483c159379d80c7a2b329572c3f9ae Mon Sep 17 00:00:00 2001 From: jirhiker Date: Tue, 23 Sep 2025 14:04:30 +0000 Subject: [PATCH 06/47] Formatting changes --- transfers/transfer.py | 2 ++ transfers/util.py | 1 + 2 files changed, 3 insertions(+) diff --git a/transfers/transfer.py b/transfers/transfer.py index 9515a86e5..c00b13243 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -113,6 +113,7 @@ def transfer_all(sess, limit=100): # if init or cleanup_wells_flag: # cleanup_wells(sess) + @timeit def main_transfer(): @@ -120,6 +121,7 @@ def main_transfer(): with session_ctx() as sess: transfer_all(sess, limit=limit) + if __name__ == "__main__": main_transfer() diff --git a/transfers/util.py b/transfers/util.py index 9bc55c630..5af2777ae 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -51,6 +51,7 @@ def write(self, buf): def flush(self): pass + # todo: setup of logging should be moved to function log_filename = f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" From c73f881e40b7109358fa9fe11d5e2d69cb9db193 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 08:07:29 -0600 Subject: [PATCH 07/47] feat: rename main_transfer endpoint to transfer_all for clarity --- transfers/entrypoint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transfers/entrypoint.py b/transfers/entrypoint.py index 77d9d99ac..2ce37d0d1 100644 --- a/transfers/entrypoint.py +++ b/transfers/entrypoint.py @@ -104,8 +104,8 @@ async def _cleanup_wells(session: session_dependency): cleanup_wells(session) -@app.post("/main_transfer") -async def _(session: session_dependency): +@app.post("/transfer_all") +async def _transfer_all(session: session_dependency): message("TRANSFERRING ALL") transfer_all(session) From 49d5a79063755523134db4fe667c2c45a1c88b6d Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 08:27:14 -0600 Subject: [PATCH 08/47] feat: rename main_transfer endpoint to transfer_all for clarity --- transfers/entrypoint.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/transfers/entrypoint.py b/transfers/entrypoint.py index 2ce37d0d1..f332c1665 100644 --- a/transfers/entrypoint.py +++ b/transfers/entrypoint.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== -from fastapi import FastAPI +from fastapi import FastAPI, BackgroundTasks from core.dependencies import session_dependency from transfers.asset_transfer import transfer_assets_testing @@ -105,9 +105,10 @@ async def _cleanup_wells(session: session_dependency): @app.post("/transfer_all") -async def _transfer_all(session: session_dependency): +async def _transfer_all(background_tasks: BackgroundTasks, session: session_dependency): message("TRANSFERRING ALL") - transfer_all(session) + background_tasks.add_task(transfer_all, session) + return {"message": "Task started"} @app.get("/health") From e5508cb2bd47f134423569749393f48caeac3aa7 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 09:50:14 -0600 Subject: [PATCH 09/47] feat: update Procfile to run main_transfer script directly --- Procfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 82b4db951..d6e928f89 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,2 @@ -web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker +#web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker --timeout 3600 +web: python main_transfer.py From b3997d1c01173a8732c146470547d033af71da94 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 09:54:17 -0600 Subject: [PATCH 10/47] feat: update Procfile to run main_transfer script directly --- Procfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Procfile b/Procfile index d6e928f89..00bd4a2dc 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1 @@ -#web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker --timeout 3600 web: python main_transfer.py From 3ce9e24e9e62bbf7687320cabfa7d01e631b9d18 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 10:14:41 -0600 Subject: [PATCH 11/47] feat: update Procfile to use python3 for main_transfer script --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 00bd4a2dc..b180e2315 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: python main_transfer.py +web: python3 transfers/main_transfer.py From 2eb3c11d3daa2138923ce856bf11f6990c11ddd0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 10:42:09 -0600 Subject: [PATCH 12/47] feat: update Procfile to use python3 for main_transfer script --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index b180e2315..539e0606e 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: python3 transfers/main_transfer.py +web: python3 transfers/transfer.py From b57864d3225c127593348419833689f1d06c685d Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 11:08:40 -0600 Subject: [PATCH 13/47] feat: update Procfile to use python3 for main_transfer script --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 539e0606e..2486669cb 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: python3 transfers/transfer.py +web: python3 -m transfers.transfer From 96d826d4fe1a73633c661e042d17aebdfbc0e05d Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 11:29:39 -0600 Subject: [PATCH 14/47] feat: update main_transfer to use environment variable for transfer limit --- transfers/entrypoint.py | 119 ---------------------------------------- transfers/transfer.py | 3 +- 2 files changed, 2 insertions(+), 120 deletions(-) delete mode 100644 transfers/entrypoint.py diff --git a/transfers/entrypoint.py b/transfers/entrypoint.py deleted file mode 100644 index f332c1665..000000000 --- a/transfers/entrypoint.py +++ /dev/null @@ -1,119 +0,0 @@ -# =============================================================================== -# Copyright 2025 ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== - -from fastapi import FastAPI, BackgroundTasks - -from core.dependencies import session_dependency -from transfers.asset_transfer import transfer_assets_testing -from transfers.contact_transfer import transfer_contacts -from transfers.group_transfer import transfer_groups -from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata -from transfers.thing_transfer import ( - transfer_met, - transfer_ephemeral_stream, - transfer_perennial_stream, - transfer_springs, -) -from transfers.transfer import message, main_transfer, transfer_all -from transfers.waterlevels_transfer import transfer_water_levels -from transfers.well_transfer import transfer_wells, cleanup_wells - -app = FastAPI(title="Transfer Service") - - -@app.post("/wells") -async def wells( - session: session_dependency, - start_index: int, - limit: int = 25, -): - results = transfer_wells(session, start_index=start_index, limit=limit) - return results - - -@app.post("/spring") -async def _(session: session_dependency, limit: int = 25): - message("TRANSFERRING SPRINGS") - transfer_springs(session, limit) - - -@app.post("/perennial_stream") -async def _(session: session_dependency, limit: int = 25): - message("TRANSFERRING PERENNIAL STREAMS") - transfer_perennial_stream(session, limit) - - -@app.post("/ephemeral_stream") -async def _(session: session_dependency, limit: int = 25): - message("TRANSFERRING EPHEMERAL STREAMS") - transfer_ephemeral_stream(session, limit) - - -@app.post("/met") -async def _(session: session_dependency, limit: int = 25): - message("TRANSFERRING METEOROLOGICAL") - transfer_met(session, limit) - - -@app.post("/contacts") -async def _(session: session_dependency): - message("TRANSFERRING CONTACTS") - transfer_contacts(session) - - -@app.post("/waterlevels") -async def _(session: session_dependency): - message("TRANSFERRING WATER LEVELS") - transfer_water_levels(session) - - -@app.post("/link_ids") -async def _(session: session_dependency): - message("TRANSFERRING LINK IDS") - transfer_link_ids(session) - transfer_link_ids_welldata(session) - - -@app.post("assets") -async def _transfer_assets(session: session_dependency): - message("TRANSFERRING ASSETS") - transfer_assets_testing(session) - - -@app.post("/groups") -async def _transfer_groups(session: session_dependency): - message("TRANSFERRING GROUPS") - transfer_groups(session) - - -@app.post("/cleanup_wells") -async def _cleanup_wells(session: session_dependency): - cleanup_wells(session) - - -@app.post("/transfer_all") -async def _transfer_all(background_tasks: BackgroundTasks, session: session_dependency): - message("TRANSFERRING ALL") - background_tasks.add_task(transfer_all, session) - return {"message": "Task started"} - - -@app.get("/health") -async def health(): - return {"status": "ok"} - - -# ============= EOF ============================================= diff --git a/transfers/transfer.py b/transfers/transfer.py index c00b13243..671abff25 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import os import time from dotenv import load_dotenv @@ -117,7 +118,7 @@ def transfer_all(sess, limit=100): @timeit def main_transfer(): - limit = 100 + limit = os.environ.get('TRANSFER_LIMIT', 1000) with session_ctx() as sess: transfer_all(sess, limit=limit) From e3e67b0d665967c46d038ed16069a0fd008f5e5b Mon Sep 17 00:00:00 2001 From: jirhiker Date: Tue, 23 Sep 2025 17:30:22 +0000 Subject: [PATCH 15/47] Formatting changes --- transfers/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 671abff25..3f89fffd8 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -118,7 +118,7 @@ def transfer_all(sess, limit=100): @timeit def main_transfer(): - limit = os.environ.get('TRANSFER_LIMIT', 1000) + limit = os.environ.get("TRANSFER_LIMIT", 1000) with session_ctx() as sess: transfer_all(sess, limit=limit) From 04c9e813b1cd1412476aac1cbc30f6e15e5d7e9a Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 11:35:48 -0600 Subject: [PATCH 16/47] feat: update main_transfer to use environment variable for transfer limit # Conflicts: # transfers/transfer.py --- transfers/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 3f89fffd8..d3905b74d 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -118,7 +118,7 @@ def transfer_all(sess, limit=100): @timeit def main_transfer(): - limit = os.environ.get("TRANSFER_LIMIT", 1000) + limit = int(os.environ.get('TRANSFER_LIMIT', 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) From d9e7137998d19571395e89220b5b5cc53b459cce Mon Sep 17 00:00:00 2001 From: jirhiker Date: Tue, 23 Sep 2025 17:36:14 +0000 Subject: [PATCH 17/47] Formatting changes --- transfers/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index d3905b74d..1c094753e 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -118,7 +118,7 @@ def transfer_all(sess, limit=100): @timeit def main_transfer(): - limit = int(os.environ.get('TRANSFER_LIMIT', 1000)) + limit = int(os.environ.get("TRANSFER_LIMIT", 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) From 951f0b749a4d68e014799d059b5a8c856c8d56d4 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 19:40:38 -0600 Subject: [PATCH 18/47] feat: add filtering functions for valid measuring agencies and well data sources --- transfers/data/valid_measuring_agency.csv | 19 + transfers/data/valid_welldata_datasources.csv | 659 ++++++++++++++++++ transfers/transfer.py | 8 +- transfers/util.py | 20 + transfers/waterlevels_transfer.py | 3 +- transfers/well_transfer.py | 15 +- 6 files changed, 711 insertions(+), 13 deletions(-) create mode 100644 transfers/data/valid_measuring_agency.csv create mode 100644 transfers/data/valid_welldata_datasources.csv diff --git a/transfers/data/valid_measuring_agency.csv b/transfers/data/valid_measuring_agency.csv new file mode 100644 index 000000000..fa600aad4 --- /dev/null +++ b/transfers/data/valid_measuring_agency.csv @@ -0,0 +1,19 @@ +"MeasuringAgency","valid" +"Bayard",NO +"Bernalillo Cty",NO +"BLM",NO +"NESWCD",NO +"NMBGMR",Yes +"NMED",NO +"NMISC",NO +"NMOSE",NO +"NMT","Yes" +"NPS",Yes +"OSWCD",NO +"PVACD",NO +"SFC",NO +"SNL",Yes +"TSWCD",Yes +"TWDB",NO +"USFS",Yes +"USGS",NO diff --git a/transfers/data/valid_welldata_datasources.csv b/transfers/data/valid_welldata_datasources.csv new file mode 100644 index 000000000..24145751b --- /dev/null +++ b/transfers/data/valid_welldata_datasources.csv @@ -0,0 +1,659 @@ +"DataSource","valid" +"(Author?), 1998, Geohydrology of River Bend Subdivision Taos County, New Mexico, Glorieta Geoscience, Inc. Appendix A. (Full report filed w/TV-121).","Yes" +"1) Author?, 1995, Town of Taos San Juan/Chama Diversion Proj., Phase 2, Vol. 1, Production Well & Observation Well Installation, Testing, and Determination of Aquifer Coefficients, GGI. 2) Water Level","Yes" +"1) Bernalillo County, Sara Chudnoff; 2) San Pedro Creek Estates HOA","Yes" +"1) BLM - Part of the Capitan Aquifer Observation-Well Network","Yes" +"1) BLM, 2) Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38, 3) Sandia Report SAND2018-12018 NM Permian Basin","Yes" +"1) BLM, 2) Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38.","Yes" +"1) Discussion of Geology, Hydrogeol., & H2O Quality of Tailings Area, Molycorp Facility, Taos Cty, NM, 1995, South Pass Resources","Yes" +"1) Discussion of Geology, Hydrogeol., & H2O Quality of Tailings Area, Molycorp Facility, Taos Cty, NM, 1995, South Pass Resources 2) Pre-Design Groundwater Investigation at the Tailing Facil. ARCADIS","Yes" +"1) Drakos, P., Riesterer, J., Lazarus, J., 2007, Drilling and Completion Report, Rio Pueblo 3200 Feet Deep Production Well (RG-37303-S), Taos, NM, GGI (Filed w/TV-171). 2) Water Level Monitoring Prgm.","Yes" +"1) GW3: Geology and Ground-Water Resources of Eddy County, NM (1952) G.E. Hendrickson & R.S. Jones, NM Bureau of Geology; 2) USGS","Yes" +"1) Lowry, T.S., et al. (2018) Water Resource Assessment in the NM Permian Basin, Sandia Report SAND2018-12018, 2) 3) Sandia Natl. Labs Report SAND2021-1869","Yes" +"1) Lowry, T.S., et al. (2018) Water Resource Assessment in the NM Permian Basin, Sandia Report SAND2018-12018, 2) Sandia Natl. Labs Report SAND2021-1869","Yes" +"1) Myers, Everheart, Wilson (1994) Geohydrol. of San Agustin Basin, Alamosa Creek Basin upstream from Monticello Box & Upper Gila Basin in parts of Catron, Socorro & Sierra Cty, NM, USGS WRI 94-4125","Yes" +"1) Sandia National Labs/ BLM Wells, 2) Sandia Natl. Labs Report SAND2021-1869","Yes" +"1) Turner, B., 1996, Water Availability for the El Mirador Subdvn, Taos Cty, NM, Turner Environ. Consult., Appd 2&3 (filed w/TV-229). 2) Vista del Valle report (filed w/TV-153). 3) Drakos, P. 2005, Ge","Yes" +"1) USGS",NO +"1) USGS - WL; 2) Brandvold - WQ","Yes" +"1) USGS - WLs 2) Brandvold -WQ","Yes" +"1) USGS - WLs 2) Brandvold -WQ, 3) OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"1) USGS 2) BLM, 3) Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38, 4) Sandia National Labs SAND2018-12018 NM Permian Basin","Yes" +"1) USGS 2) David Chace of Hydro Resolutions","Yes" +"1) USGS 2) Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38.","Yes" +"1) USGS 2) San Pedro Creek Estates HOA","Yes" +"1) USGS 2) Sandia National Labs/ BLM Well, 3) Sandia Natl. Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin","Yes" +"1) USGS 2) Sandia National Labs/BLM Well","Yes" +"1) USGS and 2) Data from Lynn Brandvold with NMBGMR chemistry lab, 3) OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"1) USGS, 2) BLM, 3) Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38, 4) Sandia Report SAND2018-12018 NM Permian Basin","Yes" +"1) USGS, 2) Circular 26: Water Well Records and Well Water Quality in Southwestern San Agustin Plains, Catron Cty, NM (1954) Bushman, F.X. and Valentine, C.P., NMBGMR","Yes" +"1) USGS, 2) OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"1) USGS, 2) Sandia National Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin","Yes" +"1) USGS, 2) Sandia Natl. Lab Report SAND2018-12018 Water Resource Assessment in the NM Permian Basin.","Yes" +"1) USGS, 2) Sandia Natl. Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin","Yes" +"1) USGS, 2) Sandia Natl. Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin, 3) GW3: Geol. & Groundwater Res. of Eddy Cty. (1952) Hendrickson & Jones, NMBGMR, 4) SNL SAND2021-1869","Yes" +"1) USGS, 2) Sandia Natl. Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin, 3) Sandia Natl. Labs Report SAND2021-1869","Yes" +"1) USGS, 2) Sandia Natl. Labs Report SAND2021-1869","Yes" +"1) Well depth and screen info. from spreadsheet from mayor of Questa. 2) Pre-design groundwater investigation at the tailing facility - Final work plan. April 24, 2013, Arcadis.","Yes" +"AGW Consultants, 12/85, Geohydro of R.San Marcos Property; Balleau GWA, 1999; field check","Yes" +"AGW Consultants, 12/85, Geohydrology of Rancho San Marcos","Yes" +"AGW Consultants, May 1984, Hydrogeology of the Santa Fe Downs Resort Area Near Santa Fe, Santa Fe County, NM","Yes" +"AGWC, Inc., 1983, Hydrogeology of the Great Cloud Zen Center, Oct 1983","Yes" +"Also see: Drakos, P., 2007, Geohydrology of the Estancias Atalaya Subdivision, Taos County, New Mexico, Glorieta Geoscience, Inc. (Filed w/TV-103).","Yes" +"American Ground Water Consultants, 1983, Hydrogeology of the Villa Marika Property, October 1983.","Yes" +"AMP visited this well on 5/25/2022 but we didn't do a well inventory.","Yes" +"ARCADIS Project B0046795.0019, Early Design Actions-Pre-Design Tailing Facility Groundwater Invest. Results from Monitoring Wells MW-43, MW-44 and MW-45 (May 1, 2015).","Yes" +"ARCADIS well/lithologic log","Yes" +"Author (?), 1998, Geohydrology of River Bend Subdivision, Taos County, New Mexico, Glorieta Geoscience, Inc. Also in NMGS Gdbk 55 pg 396.","Yes" +"Balleau Groundwater, 2001, CDEX1 Completion Report","Yes" +"Balleau Groundwater, 2002, CDPROD1 Completion Report, 2/02","Yes" +"Balleau Groundwater, Inc., Oct 2007, Injection Demonstration Well -- Well Completion Report","Yes" +"Balleau Groundwater, Inc., Oct 2007, Observation Well A -- Well Completion Report","Yes" +"Balleau Groundwater, Inc., Oct 2007, Observation Well B -- Well Completion Report","Yes" +"Balleau Groundwater, Inc., written communication, 11/27/2007","Yes" +"Balleau GW, 2002, CDPROD Completion Report; well log","Yes" +"Balleau GWA, 1999; Balleau GWA","Yes" +"Balleau GWA, 1999; field check","Yes" +"Balleau GWA, 1999; field check; Jenkins, 1977, Geohydro Investigation of Turquoise Trail Subdivision","Yes" +"Bernalillo County",NO +"Bernalillo County, Sara Chudnoff",NO +"Bernalillo County, Sara Chudnoff; Screen interval from DBSA Campbell Ranch report",NO +"Bjorklund, L.J. (1957) Reconnaissance of Groundwater Conditions in the Crow Flats Area Otero Cty, NM, OSE Tech Rep 8.","Yes" +"Black & Veatch Consulting Engineers, 1978, Construction and Testing Report Buckman Well No. 2; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; C. Borchert, written communication, 3/23/09","Yes" +"Black & Veatch Consulting Engrs, 1978, Construction and Testing Rept Buckman Well No. 1; NMED DWB (1999) via Jemez y Sangre Database (2000); Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis","Yes" +"Bob Hubbard","Yes" +"Bureau scientists performed a pump test on this well.","Yes" +"C. Borchert, written comm.; WATERS","Yes" +"Carrol Wood, well owner.","Yes" +"Cathie.","Yes" +"CB00107524CCA","Yes" +"CDM completion log to NMED SW Bureau","Yes" +"CDM well completion log to NMED SW Bureau; written communications from NMED SolidWaste Bureau","Yes" +"CH2M Hill report to NMED UST Bureau","Yes" +"CH2M Hill water level measurements at WWTP (entered into db).","Yes" +"Charles Heaton, Sinagua Consultants, 1999, Well Hydrology Report Elmer Garcia Property, Aug 1999; County Records; well log; field check","Yes" +"Charles Walker","Yes" +"Chemistry data from EPA via NM Health Dept.",NO +"Chemistry data from Lynn Brandvold with NMBGMR chemistry lab. NMBGMR visited the site after chem. data input.","Yes" +"Chemistry from EPA via NM Health Dept.",NO +"Chemistry from Matt Sophy","Yes" +"Chemistry in Excel spreadsheet (See documents in Aquifer_mapping\BrackishWater\BW_database\BOR-Desal-Wells)","Yes" +"Circular 26: Water Well Records and Well Water Quality in Southwestern San Agustin Plains, Catron Cty, NM (1954) Bushman, F.X. and Valentine, C.P., NMBGMR","Yes" +"City of Santa Fe; OSE records. NMED DWB (1999) via Jemez y Sangre Database (2000). Shomaker, J.W., 1998, Sustainable Ground-Water Production from City Well Field, Santa Fe, NM, 4/98; CDM, Jan 2001","Yes" +"City of Santa Fe; OSE records. NMED DWB (1999) via Jemez y Sangre Database (2000); Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis","Yes" +"City of Santa Fe; OSE records; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis","Yes" +"Claudia Borchert, 2002, MS Thesis","Yes" +"Compiled by Susan VonGonten, NMED UST Bureau","Yes" +"Consulting Professionals, Inc, Hydrogeologic Rept La Tierra Subdn, Phase 4, Santa Fe Cnty, Dec1978","Yes" +"Consulting Professionals, Inc., VeneKlasen, 1975, Hydrogeologic Report for La Tierra Subdivision, 6/75; well log","Yes" +"Cooper, 1991, Geohydro Rept for Country Boarding Kennel, 5/1991","Yes" +"Cooper, 1995, Garcia Ranch report; Wilson, 1978, SFC water plan Table 1-28; field check by Johnson; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis.","Yes" +"Cooper, 1995, rept to Garcia Ranch; Kuckleman, 2003","Yes" +"Cooper, D. R., 1995, Geohydrology Report for Cottonwood Ranch Subdn, Santa Fe Cty, NM, April 1995","Yes" +"Cooper, D. R., 1998, Geohydrology for WK Jones, Santa Fe Cnty, Oct 1998","Yes" +"Cooper, D. R., Geohydrologic Report for Rancho De Los Cuervos, Santa Fe Cnty, Sept1986; Berman Tract","Yes" +"Cooper, D. R., Geohydrology Report for Frank M Gallegos & Andrew M Leyba, Santa Fe Cnty, Oct, 1999","Yes" +"Cooper, D. R., Geohydrology Report for Friendly Construction, Inc., Santa Fe Cnty, July 1994","Yes" +"Cooper, D. R., Geohydrology Report for Tano Vista Grande, Santa Fe Cnty, NM, Nov 1996","Yes" +"Cooper, D. R., Geohydrology Rept for Marvin Pollock & Bettina Lancaster, Santa Fe Cnty, May 1994","Yes" +"Cooper, D.R., Geohydrology Rept for El Prado Subdn, Santa Fe Cnty, Sept 1994","Yes" +"Cooper, D.R., Santa Fe Design Assoc, Geohydrology Rept for Rancho Oso Loco, Santa Fe Cnty, 1985","Yes" +"Cooper, Dennis R., Geohydrology for Rancho De Los Ninos, Santa Fe Cnty, NM, April 1999; Rick Borrego","Yes" +"Cooper, Dennis R., Geohydrology Report for Bob Shipp, Santa Fe County, NM, June 1995","Yes" +"Cooper, Dennis R., Geohydrology Report for Heartstone Development LLC, Dec 2001","Yes" +"Cooper, Dennis R., Geohydrology Report for James B. Alley Jr., Santa Fe Cnty, NM, March 1995","Yes" +"Cooper, Dennis R., Geohydrology Report for Lawrence Tuchman, Santa Fe Cnty, Oct 1994; Frost & Assoc., Santa Fe County WQ Monitoring, June 1996","Yes" +"Cooper, Dennis R., Geohydrology Report For Los Suenos, Santa Fe County, NM, April 1994","Yes" +"Cooper, Dennis R., Geohydrology Rept for Jeffrey Jacobs & Thad Bowman, Santa Fe Cnty, Jan 1996","Yes" +"Cooper, Dennis R., July 1995, Analysis of water levels in wells on Garcia Ranch, unpub consult rept","Yes" +"Cooper, Dennis R., July 1995, Unpub report to Garcia Ranch; well log; WATERS; USGS field notes","Yes" +"Cooper, Dennis, 1990, Geohydrology Report for San Juan Complex Partnership Ltd., Sept 1990; Field check by Johnson; well log; SFO files","Yes" +"Cooper, Dennis, 1993, Geohydrology Rept for Neighbors, Inc., Santa Fe Cnty, Dec 1993; well log Joe Briscoe","Yes" +"Cooper, Dennis, 2000, Geohydro Rept for Roybal Subdivision, Santa Fe Cnty, NM, Oct 2000; field check","Yes" +"Cooper, Dennis, Geohydrol Rept for Hacienda Del Cerezo, Santa Fe Cnty, NM, Jan 1994; well logs","Yes" +"Cooper, Dennis, Geohydrology Rept for B. Howard & J.Morris, Santa Fe Cnty, NM, Dec 1994","Yes" +"Cooper, Dennis,1991, Geohydrology Rept for Sheila Cooper; field check by Johnson","Yes" +"Cooper, Geohydro Report for Land Ventures, LLC, 6/2001; field check by Johnson; well log","Yes" +"Cooper, Geohydrology Rept for Welsh Family Ltd Partnership, Santa Fe Cnty, March 1995","Yes" +"Corbin Consulting, Inc., 2005, Geohydrology Report (RG-27728-S) Longanecker Property. March 8, 2005","Yes" +"Corbin Consulting, Inc., 2005, Geo-Hydrology Report McMillan Subdivision, June 6, 2005","Yes" +"Corbin, J., 2004, Constant Property Geohydrology Report, Corbin Consulting Inc. June 3, 2004. Field checked.","Yes" +"Corbin, James, 2005, Reconnaissance Report McMillan Property. Feb. 28, 2005 (EB-618)","Yes" +"Corbin, James, 2005, Reconnaissance Report McMillan Property. Feb. 28, 2005; (EB-618)","Yes" +"Corbin, James, 2005.","Yes" +"County records; WATERS; drilling log","Yes" +"County records; well log","Yes" +"CT890 also at same site, cinfusing records not sure which well is correct, this one has a log","Yes" +"Cuttings from NMBG. Bill White has cuttings. Geophysics: gamma ray, neutron, 3-arm-caliper, resistivitity, single pt resistance, SP, temperature profile","Yes" +"Dames & Moore, Inc., 1995, Geohydroloy Report Komis Estates for Southwest Surveying Co., Inc., Dec. 1995; J. Corbin personal communication, 2005","Yes" +"Daniel B. Stephens & Associates report on Campbell Ranch subdivision wells","Yes" +"Daniel B. Stevens & Associates consultant report (AMP\data\datasets\Campbell Ranch Bernalillo County)","Yes" +"Darr, M.J., 2006, Geohydrologic Investigation Report: Proposed ""Ocotillo"" Subdivision, Taos County, NM. MJDarrconsult, Inc / Hydroscience Assoc. Inc, 2006, Analysis of Aquifer Test Run Using RG-87080","Yes" +"Darr, M.J., 2008, Geohydrologic Investigation Report: Proposed Los Llanos Subdivision, Taos County, New Mexico, MJDarrconsult, Inc.","Yes" +"Darr, Michael J., 2007, Geohydrologic Investigation Report: Proposed ""Golf Country"" Subdivision, Taos County, New Mexico, MJDarrconsult, Inc.","Yes" +"Data from Groundwater Report 2: Geology & Groundwater Resources of San Miguel County, NM. (1951) NMBGMR by R.L. Griggs and G.E. Hendrickson.","Yes" +"Data from Lynn Brandvold with NMBGMR chemistry lab.","Yes" +"Data from Lynn Brandvold with NMBGMR chemistry lab., OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"Data from owner well testing via NMBGMR water lab.","Yes" +"David Chace of Hydro Resolutions","Yes" +"David Jenkins, Geohydrology of the Los Vaqueros Subdn, Santa Fe Cnty, May 1983","Yes" +"David N Jenkins, Geohydrolgoy of the Las Dos, Phase II Area, Santa Fe County, NM, October 1982","Yes" +"David N Jenkins, Geohydrologic Conditions at the San Juan Residences, Santa Fe Cnty, April 1982","Yes" +"David N Jenkins, Geohydrology of the Los Vaqueros Subdn, Santa Fe Cnty, May 1983","Yes" +"David N Jenkins, Geohydrology of the Los Vaqueros Subdn, Santa Fe Cnty, May 1983;","Yes" +"David N Jenkins, Geohydrology of the Los Vaqueros Subdn, Santa Fe Cnty, May 1983; Steinberger log of R1 test hole, K.Summers files","Yes" +"David N Jenkins, Geohydrology of the Los Vaqueros Subdn, Santa Fe Cnty, May 1983; WK Summers project files","Yes" +"David N Jenkins, Geohydrology of the Vista Subdivision, Santa Fe County, Dec 1979","Yes" +"David N Jenkins, Geohydrology of the Vista Subdivision, Santa Fe County, Dec 1979; Mourant; Spiegel & Baldwin, p 241","Yes" +"David N Jenkins, Geohydrology of the Vista Subdivision, Santa Fe County, Dec 1979; WATERS","Yes" +"David N. Jenkins, Geohydrologic Conditions at the Thorpe Condominiums, Santa Fe Cnty, Nov 1982","Yes" +"David Updegraff & William Lyons, 1978, Geohydrology of the Hyde Park Estates Unit 3 Subdiv'n, Aug 1978","Yes" +"DBStephens report to NMED UST Bureau","Yes" +"depth made up by GCR based on nearby wells, no actual data","Yes" +"depth made up by GCR, no actual data, assume similar to well K-14 (UC 30)","Yes" +"Discussion of Geology, Hydrogeology, and Water Quality of the Tailings Area, Molycorp Facility, Taos County, NM, 1995, South Pass Resources, Inc.","Yes" +"Drakos, P. & Hodgins, M., 2000, Drilling and Testing Report, Bureau of Reclamation 2000-Feet Deep Nested Piezometer/Exploratory Well (BOR #1), Taos, NM, GGI. Also, Water Level Monitoring Prgm. 2002-2","Yes" +"Drakos, P. and Hodgins, M., 2000, Drilling and Testing Report, Bureau of Reclamation 2000-Feet Deep Nested Piezometer/Exploratory Well (BOR #1), Taos, NM, Glorieta Geoscience, Inc. Also, Water Level","Yes" +"Drakos, P., 2007, Geohydrology of the Estancias Atalaya Subdivision, Taos County, New Mexico, Glorieta Geoscience, Inc.","Yes" +"Drakos, P., et al., 2004, Hydrologic Characteristics of Basin-Fill Aquifers in the Southern San Luis Basin, NM, NMGS Guidebook 55, pg. 391.","Yes" +"Drakos, P., Hodgins, M., 2000, Drilling & Testing Report, Bureau of Reclamation 2000-Ft. Deep Nested Piezometer/Exploratory Well (BOR #1), Taos, NM, GGI. 2) Water Level Monitoring Prgm. 2002-2007, GGI","Yes" +"Drakos, P., Hodgins, M., Lazarus, J. & Riesterer, J, 2002, Drilling & Testing Report, BOR 2000-Ft. Deep Nested Piezometer/Expl. Well (BOR-2) & 2100-Ft. Deep Production Well (BOR-3), Paseo del Canon We","Yes" +"Drakos, P., Hodgins, M., Lazarus, J. and Riesterer, J, 2002, Drilling & Testing Report, BOR 2000-Feet Deep Nested Piezometer/Exploratory Well (BOR-2) & 2100-Feet Deep Production Well (BOR-3), Paseo de","Yes" +"Drakos, P., Riesterer, J., and Lazarus, J., 2007, Drilling and Completion Report, Rio Pueblo 3200 Feet Deep Production Well (RG-37303-S), Taos, NM, Glorieta Geoscience, Inc. Also, Water Level Monitor","Yes" +"Driller's log; OSE network WLs","Yes" +"dubiousOSE match, depth and location are off","Yes" +"Duncan, 2004.","Yes" +"Duncan, 2004; field check by Johnson 12-22-05","Yes" +"Duncan, 2004; Frost, 1999, ElDorado Area Water & Sanitation District Project files","Yes" +"Duncan, 2004; J.Frost, 1999, Eldorado Area WQ study","Yes" +"Duncan, 2004; J.Frost, 1999, Eldorado Area WQ study.","Yes" +"Duncan, 2004; JSAI, 2001, App. #6","Yes" +"Exploratory & Shallow Well Drilling Rio Grande Watershed Study-Phase 1 San Acacia Surface Water/Groundwater Investigation (Dec. 5, 2003) S.S. Papadopulos & Associates for US Army Corps of Engineer","Yes" +"Faith Engineering, Inc., 1994, Pump Test Report for the Alto St Well, 12/5/94","Yes" +"Field check by Johnson","Yes" +"Field check by Johnson, Cruz, Frost; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; D.Koning log review","Yes" +"Field check by Johnson; Cooper, 1995, report to Garcia Ranch; Borton field notes","Yes" +"Field check by Johnson; Kuckleman Pump Service, report of visit 2/14/2002","Yes" +"Field check by Johnson; Kuckleman Pump Service, report of visit 7/2/98","Yes" +"Field check by Johnson; owner info","Yes" +"Field check by Johnson; WATERS","Yes" +"Field check by Johnson; WATERS; Mourant","Yes" +"Field check by Johnson; WATERS; sample site for Anderholm, 1994 USGS WRI Report 94-4078","Yes" +"Field check by Johnson; well log","Yes" +"Field check by Johnson; well log; owner","Yes" +"Field check by Johnson; well log; SFO files","Yes" +"Field check by Johnson; well record","Yes" +"Field check by Johnson; well/owner records","Yes" +"Field check by Lyman","Yes" +"Field check with Lyman and BLM realtor Hal Knox; plat map","Yes" +"Field check, WATERS","Yes" +"Field check; owner","Yes" +"Field check; owner; well log","Yes" +"Field check; WATERS","Yes" +"Field check; WATERS. OSE Well Record.","Yes" +"Field check; WATERS.OSE Well Record.","Yes" +"Field check; well log","Yes" +"Field check; well logg","Yes" +"Field check; well record","Yes" +"Field checked","Yes" +"Field checked, WATERS","Yes" +"Field checked, Well owners recollection of well info","Yes" +"Field checked. WATERS. OSE Well Record.","Yes" +"Field checked; WATERS","Yes" +"Field checked; WATERS. OSE Well Record.","Yes" +"Field checks; WATERS; well record.","Yes" +"Finch & Petronis (JS&A), Sept. 2006, Hydrogeologic Report for the Galisteo Basin Preserve, Santa Fe County, NM, for Commonweal Conservancy, unpub. consultant report","Yes" +"Finch, S.T., Jr., 2001, Hydrogeologic Evaluation of T-255 Et. Al., Near Carrizozo, New Mexico, John Shomaker & Associates, Inc.","Yes" +"Finch, Steven T., and Melis, Erwin A., October 2008, Hydrogeologic evaluation of ground-water supply for the Spaceport America Site near Upham, NM, John Shomaker & Associates.","Yes" +"Fleming, WM, Geohydrology Report for the Matthews Property, Santa Fe County, July 26, 1991","Yes" +"Foreman","Yes" +"Frost MWB notes, 1994","Yes" +"Frost MWB Survey notes","Yes" +"Frost, 1996, Santa Fe Cnty Ground Water Levels, 1995-96 Results; GWSI; USGS log books; well record","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results, Jan 1996","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results, Jan 1996; field check","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results, Jan 1996; Lewis database; NMED DWB (1999) via Jemez y Sangre Database (2000)","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results, Jan 1996; Mourant","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results; Mourant; GWSI; fieldchk by Johnso","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results; Mourant; GWSI; fieldchk by Johnson","Yes" +"Frost, 1996, Santa Fe County Ground Water Levels, 1995-96 Results; Mourant; GWSI; well record","Yes" +"Frost, 1999; WATERS","Yes" +"Frost, MWB Survey notes; D.Koning log review","Yes" +"Full EPA ID #TB-B011-150811-21","Yes" +"Full EPA ID #TB-B023-150814-021","Yes" +"Full EPA ID #TD-D001-150812-21","Yes" +"Full EPA ID #TD-D003-150812-21","Yes" +"Garrabrant, L.A., 1993, Water resources of Taos County, NM: USGS WRIR 93-4107.","Yes" +"Geohydrologic Assessment of the Augustin Plains Ranch Area, NM (Tech. Memorandum), March 2, 2012, Geoscience. Cuttings archived in our core shed.","Yes" +"Geohydrologic Assessment of the Augustin Plains Ranch Area, NM (Tech. Memorandum), March 2, 2012, Geoscience. Pump test from driller (JSAI). Cuttings/chip boards archived in our core shed.","Yes" +"Geohydrology Assoc., Inc. 1983, Geohydrology of Rancho Viejo Prop, SFC, NM, Feb 1983, LMCoons","Yes" +"Geohydrology Assoc., Inc. 1983, Geohydrology of Rancho Viejo Prop, SFC, NM, Feb 1983, LMCoons; map","Yes" +"Geohydrology Assoc., Inc. lithologic log to NMED SW Bureau; field check by Johnson","Yes" +"Geohydrology Assoc., Inc., 1988, Hydrogeologic Investigation of Cottonwood Estates, 2//88; WATERS","Yes" +"Geophysics: dual induction guard, dual spaced neutron,microlog, high-resol temp log. WQ: Sorrell, John D. and Banet, Christopher, Karavas Tract Exploratory Well, Nov. 1993, Bureau of Indian Affairs.","Yes" +"GGI","Yes" +"GGI report (has WQ sample). Requested from P. Drakos; have not received response.","Yes" +"GGI report for NMED UST Bureau","Yes" +"GGI report to NMED UST Bureau","Yes" +"GGI reports: 1) 2001 report on RP-2500 in Town of Taos, 2) Water Level Monitoring","Yes" +"GGI, Geohydrology of the Woods Property, Santa Fe Cnty, NM,3/19/92; Sangre de Cristo Estates rept","Yes" +"GGI, 100-Yr Water Availability Santa Fe Metro Center, 12/01; AND La Cienega de Santa Fe Proj, 10/91","Yes" +"GGI, 1988, Geohydrology of the Picture Rock Development Co. Property, Santa Fe, May, 1988.","Yes" +"GGI, 1989, Geohydrology of the Baird Joint Venture Property, Taos County, NM, Glorieta Geoscience, Inc.","Yes" +"GGI, 1989, Reconnaissance Geohydrology of the Lumpkins Property, Santa Fe County, NM, August 1989","Yes" +"GGI, 1990, Geohydrology of the Bajo Del Cielo Subdivision, Santa Fe County, NM. Dec. 1990","Yes" +"GGI, 1990, Geohydrology of the Beaver and Henry Properties, Santa Fe Cnty, Aug 1990","Yes" +"GGI, 1991, Geohydrology of the Walker Property, Santa Fe, May, 1991.","Yes" +"GGI, 1995, Geohydrology of the Santa Fe Opera Tract Prop, Santa Fe Cnty, 3/95; G.Ellis; well log","Yes" +"GGI, 1995, Geohydrology of the Vereda Serena Property, Santa Fe, October, 1995.","Yes" +"GGI, 1995, Production well and observation well installation, testing, and determination of aquifer coefficients: unpub consultant report for Town of Taos, March 1995","Yes" +"GGI, 1998, Ground Water Conditions in the Vicinity of the Gonzales Tract San Marcos Arroyo Santa Fe County, NM, Dec. 1998","Yes" +"GGI, 2001, Geohydrologic report for the Village at Eldorado, July, 2001.","Yes" +"GGI, 2001, Vista del Valle Subdivision Report, Taos County, New Mexico","Yes" +"GGI, 2002, Addendum To: Reconnaissance Geohydrologic Characterization of Tesuque Ridge, 8/28/2002","Yes" +"GGI, 2002, Reconnaissance Geohydrologic Characterization of Tesuque Ridge Subdivn, 7/17/02","Yes" +"GGI, 2003, Geohydrologic Report for the Luna Rosa LLC Equestrian Center Property, July 2003.","Yes" +"GGI, 2003, Reconnaissance Geohydrology report of the Clements Property, Santa Fe, April, 2003.","Yes" +"GGI, 2003, Reconnaissance Water Availability Assessment of the Santa Fe Waldorf School - High School","Yes" +"GGI, 2004, Geohydrology of the Estancia Subdivision, Santa Fe County, December, 2004","Yes" +"GGI, 2004, Reconnaissance Geohydrology Report for the Beaty Property, Santa Fe Cty, Dec. 22, 2004.","Yes" +"GGI, Chapter VI Geohydrology of the La Cienega de Santa Fe Project, Oct 1991; GGI, 100-Yr Water Availability, Downs at Santa Fe, March 2001 for Pojoaque Pueblo Development Corp","Yes" +"GGI, Geohydro of Naiche Property, 12/92; field check by Johnson; well recd","Yes" +"GGI, Geohydrologic report for the Rancho San Lucas Subdivision, Feb 2002","Yes" +"GGI, Geohydrology of La Canada Subdivision, Santa Fe County, NM, July 1985","Yes" +"GGI, Geohydrology of Pop's Convenience Store, Santa Fe County, NM, Sept 29, 1997","Yes" +"GGI, Geohydrology of the Brenner Property, Santa Fe County, NM, Aug 1990","Yes" +"GGI, Geohydrology of the Bryant Prop, Santa Fe Cnty, May 1995; GGI, San Ysidro de Tesuque, May 1990","Yes" +"GGI, Geohydrology of the Bryant Property, Santa Fe County, NM, May 1995; GGI, 1989","Yes" +"GGI, Geohydrology of the Circle Drive Compound Prop, Santa Fe Cnty, May 1991","Yes" +"GGI, Geohydrology of the Hansen Property, Santa Fe County, NM, June 1990","Yes" +"GGI, Geohydrology of the Insight Investments Property, Santa Fe Cnty, NM, 2/20/90","Yes" +"GGI, Geohydrology of the Lane Property, Santa Fe Cnty, NM, 4/29/92","Yes" +"GGI, Geohydrology of the McElvain/Patania Property, 6/92","Yes" +"GGI, Geohydrology of the McMahon Property, Santa Fe Cnty, Oct 1990","Yes" +"GGI, Geohydrology of the McMahon Property, Santa Fe Cnty, Oct 1990; GGI, Geohydrology of the Santa Fe Arroyo Hondo Vistas Subdivision, SFe County, Aug. 1989","Yes" +"GGI, Geohydrology of the Migel Property, Santa Fe Cnty, 2/90; GGI, Geohydro of McMahon Prop 10/90","Yes" +"GGI, Geohydrology of the Mountain Vista Subdivision, Santa Fe County, July 10, 2002","Yes" +"GGI, Geohydrology of the Myers Property, Santa Fe Cnty, July 1992","Yes" +"GGI, Geohydrology of the Naiche Property, Santa Fe County, NM, Dec 1992","Yes" +"GGI, Geohydrology of the Prince Property, 8/91; McElvain-Patania Prop, 6/92","Yes" +"GGI, Geohydrology of the Santa Fe Animal Shelter Site, Santa Fe County, NM, May 15, 2002","Yes" +"GGI, Geohydrology of the Santa Fe Opera Tr, Santa Fe Cnty, 3/95; GGI, San Ysidro de Tesuque, 5/90","Yes" +"GGI, Geohydrology of the Shulman Property, Santa Fe Cnty, April 1990","Yes" +"GGI, Geohydrology of the Vrtiak Property, Santa Fe Cnty, June, 1991","Yes" +"GGI, Geohydrology of the West Alameda Proj, Santa Fe Cnty, NM, Jan 1989","Yes" +"GGI, Geohydrology of the Woods Property, Santa Fe Cnty, March 19, 1992","Yes" +"GGI, Geohydrology of the Yamada Property, Santa Fe Cnty, Oct 14, 1994; GGI, Bryant Prop; WATERS","Yes" +"GGI, Geohydrology of the Yamada Property, Santa Fe Cnty, Oct, 1994; GGI, Sangre de Cr Est, Jan 1989","Yes" +"GGI, Geohydrology Rept for Rancho San Lucas, Santa Fe Cnty, NM, Feb 2002; Geophysical logs","Yes" +"GGI, January 2005 Alexsis Geohydrology report (by Meghan Hodgins).","Yes" +"GGI, Reconnaissance Geohydrology Report, Leibman Property, Santa Fe County, NM. March 23, 1994","Yes" +"GGI, Reconnaissance Geohydrology Report, Neufeld Property, Santa Fe Cnty, Nov 1988","Yes" +"Glorieta Geoscience Inc.","Yes" +"Glorieta Geoscience, Inc., 2006, Geohydrology and Water Availability for Phase 1, Santa Fe Canyon Ranch Subdivision, Santa Fe County, NM, March 22, 2006.","Yes" +"GW-1: Geology & Ground-Water Resources of the Eastern Part of Colfax Cty, NM (1948) Griggs, Roy L., NMBGMR","Yes" +"GW3: Geology and Ground-Water Resources of Eddy County, NM (1952) G.E. Hendrickson & R.S. Jones, NM Bureau of Geology","Yes" +"GW-4: Geology & Groundwater Res. of NE Socorro Cty, NM (1955) Spiegel, Zane, NM Bureau of Geology","Yes" +"GW5: Geology and Ground-Water Resources of Torrance County, NM (1957) R.E. Smith, NM Bureau of Geology","Yes" +"GW9: Groundwater Resources & Geology of Quay County, NM (1966) Charles Berkstresser, Jr. & Walter A. Mourant, NM Bureau of Geology","Yes" +"GWSI","Yes" +"GWSI/USGS well sheet; field check by Johnson","Yes" +"GWSI/USGS well sheet; field check by Johnson; D.Koning log review","Yes" +"GWSI/USGS well sheet; OFR 89-37; field check by Johnson; D.Koning log review","Yes" +"GWSI/USGS well sheet; WATERS","Yes" +"GWSI/USGS well sheet; WATERS; Cooper, 1995; field check by Johnson","Yes" +"GWSI/USGS well sheet; WATERS; field check by Johnson","Yes" +"GWSI/USGS well sheet; well record; field check by Johnson","Yes" +"GWSI/USGS well sheet; well record; field check by Johnson.","Yes" +"GWSI; D.Koning log review","Yes" +"GWSI; D.Koning log review; D.Koning log review","Yes" +"GWSI; field check by Johnson","Yes" +"GWSI; field check by Johnson; D.Koning log review","Yes" +"GWSI; field check Johnson","Yes" +"GWSI; Mourant; well record; field check by Johnson","Yes" +"GWSI; USGS log books; field check by Johnson","Yes" +"GWSI; USGS log sheets; field check by Johnson; field check by Lyman and SFC Utilities","Yes" +"GWSI; USGS log sheets; well log; field check by Johnson","Yes" +"GWSI; USGS log sheets; well record; field check by Johnson","Yes" +"GWSI; USGS log sheets; well record; field check by Johnson; field check by Lyman and SFC Utilities","Yes" +"GWSI; USGS logbooks; field check by Johnson","Yes" +"GWSI; USGS logbooks; Frost, 1996, Santa Fe County GWLs, 1995-96 Results, 1/96; K.Summers notes from 11/16/70 aquifer test; Frost, 1999, written comm.","Yes" +"GWSI; USGS well sheet; field check by Johnson","Yes" +"GWSI; USGS well sheet; well log; Johnson, 2003; VeneKlasen, 2/86, Geohydro Rept Arroyo Hondo West","Yes" +"GWSI; USGS well sheets; field check by Johnson","Yes" +"GWSI; water right declaration; field check by Johnson","Yes" +"GWSI; water right declaration; field check Johnson; Aqua Drilling well log. WQ from Longmire","Yes" +"GWSI; WATERS","Yes" +"GWSI; well log; field check by Johnson","Yes" +"GWSI; well record; field check by Johnson","Yes" +"Hall, J., 2010, Blackstone Ranch Well RG-82913 48-hr Pumping Test, Taos, NM, GGI.","Yes" +"Heaton (Sinagua Consultants), Well Hydro Report Capitol Ford Body Shop, Jan 1999","Yes" +"Hiss, W.L. (1973) Capitan Aquifer Observation-Well Network Carlsbad to Jal, NM, OSE Technical Report #38.","Yes" +"Hood, J.W. and Kister, L.R. (1962) Saline-Water Resources of NM. USGS Water-Supply Paper 1601","Yes" +"Hood, J.W. and Kister, L.R. (1962) Saline-Water Resources of NM. USGS Water-Supply Paper 1601; Bjorklund, L.J. (1957) Recon. of Groundwater Conditions in Crow Flats Area Otero Cty, NM, OSE Tech Rep 8","Yes" +"Hood, J.W. and Kister, L.R. (1962) Saline-Water Resources of NM. USGS Water-Supply Paper 1601; Bjorklund, L.J. (1957) Recon. of Groundwater Conditions in Crow Flats Area Otero Cty, NM, OSE Tech Rep 8.","Yes" +"HR-1: Geology & Ground-Water Resources of Central & Western Dona Ana County, NM (1971) King, W.E., Hawley, J.W., Taylor, A.M. and Wilson, R.P. New Mexico Bureau of Geology","Yes" +"HR5: Groundwater in the Sandia and Northern Manzano Mtns, NM (1980) Frank B. Titus, NM Bureau of Geology","Yes" +"Huff, G.F. (1996) Analysis of Ground-water data for selected wells near Holloman AFB, NM, 1950-1995. USGS WRIR-96-4116.","Yes" +"Hydrogeology and Underflow Estimates in the Vicinity of the Proposed Guadalupe Mountain Tailings Facility, Dames & Moore, 1988.","Yes" +"Hydrogeology and Underflow Estimates in the Vicinity of the Proposed Guadalupe Mountain Tailings Facility, Dames & Moore, March 17, 1988.","Yes" +"Hydrologic and hydrogeologic analysis for the Guadalupe Mtn. ground-water discharge plan. June 1984, Water Resources Associates, Inc.","Yes" +"Hydroscience Associates, Inc., 2006, Analysis of Aquifer Test Run Using Well RG-87082, Taos County, New Mexico. Lauren Sherson established USGS Site ID for the NGWMN project (5/29/18 - KP).","Yes" +"Info. from technical memorandums on construction\development of supply & observation wells. (See documents in Aquifer_mapping\BrackishWater\BW_database\BOR-Desal-Wells)","Yes" +"Info. from technical memorandums on construction\development of supply & observation wells. Chemistry in Excel spreadsheet (See documents in Aquifer_mapping\BrackishWater\BW_database\BOR-Desal-Wells)","Yes" +"Information as personal communication from GGI","Yes" +"Inventoried previously on 4/21/2015 as part of Clovis-Portales project","Yes" +"Inventoried previously on 6/2/2015 as part of Clovis-Portales project","Yes" +"J. Corbin, 2007 written communication","Yes" +"J. Corbin, 2007, written communication","Yes" +"J.Frost, 1999; WATERS","Yes" +"Jemez y Sangre (2000); NMED DWB Info Sheet; WATERS","Yes" +"Jenkins, 1979, Geohydrology of the Vista Subdivn, 12/79; C.A. Coonce & Assoc., 1977, Montoya Subdivn Water Availability Study for Cipriano Martinez, 12/77","Yes" +"Jenkins, 1980, Geohydro Rept for Rancho Caballero; IN Jenkins 1982, Geohydro Las Cuadras de Ocate","Yes" +"Jenkins, 1980, Geohydro Rept for Rancho Caballero; IN Jenkins 1982, Geohydro Las Cuadras de Ocate. also Jenkins, Dec 1979, Geohydrology of the Vista Subdivision","Yes" +"Jenkins, D.L., 1978, Supplemental Geohydrologic Data for the Proposed Los Caminitos Subdivision, Phase 1, Santa Fe, NM. Sept. 1978","Yes" +"Jenkins, D.L., 1978, Supplemental Geohydrologic Data for the Proposed Los Caminitos Subdivision, Phase 1, Santa Fe, NM. Sept. 1978. OSE Well Record.","Yes" +"Jenkins, David,- Geohydrologic Investigation of the Turquoise Trail Subdivision, SF Cnty, July 1977","Yes" +"Jenkins, July 1977, Geohydrologic Investigation of Turquoise Trail Subdvn","Yes" +"Jenkins, July 1977, Geohydrologic Investigation of Turquoise Trail Subdvn; Corbin Consulting, Inc., Mar 8, 2005, Geohydrology Report Longanecker Property","Yes" +"JHA, May 2004, Well Report Buckman Wells No. 10-13, City of Santa Fe","Yes" +"John Shomaker & Assoc., April 1995, Well report Sangre de Cristo Water Company Buckman Well No. 3a; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; C.Borchert, 3/23/09, written comm.","Yes" +"John Shomaker & Assoc, 1999, Well Report: Drilling, Construction, & Testing Hickox Well No. 2, 5/99; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; D.Koning review","Yes" +"Johnson, P., et al. (2005) Hydrogeology & Water Resource Assessment of the Pueblo of Picuris, Taos County, NM. NMBGMR","Yes" +"Johnson, P., et al. (2005) Hydrogeology & Water Resource Assessment of the Pueblo of Picuris, Taos County, NM. NMBGMR.","Yes" +"JSA completion log to NMED SW Bureau; written communications from NMED SolidWaste Bureau","Yes" +"JSAI Well report for Proposed Desalination Well Field, City of Alamogordo, NM. June 2006.","Yes" +"Kendall Taylor","Yes" +"Kendall Taylor and drill log","Yes" +"Kendall Taylor; OSE well log","Yes" +"L. Sherson @ USGS determined that all but one water level taken by contractors for OSE at 334819108084801 were really taken from this location 334819108084601. Level on 9/25/2008 was at *4801 NM-15416","Yes" +"Lauren Sherson established USGS Site ID for the NGWMN project (5/29/18 - KP).","Yes" +"Lauren Sherson established USGS Site ID for the NGWMN project (5/29/18 - KP). Part of OSE network, so thought it would have been in their system already.","Yes" +"Lewis database of city wells; field check by Johnson; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis","Yes" +"Lewis database, 2001; Faith Engineering, Inc., 1994, Pump Test Report for the Alto St Well, 12/5/94; Shomaker, J.W., 1998, Sustainable Ground-Water Production from City Well Field, Santa Fe, NM, 4/98","Yes" +"Lithlog personal communication w/GGI.","Yes" +"local individuals: Layne Preslar, Bob Hubbard","Yes" +"Longmire, Patrick, July 1985, Hydrogeochemical Study Along the Santa Fe River; field check","Yes" +"Longmire, Patrick, July 1985. A Hydrogeochemical Study Along the Valley of the Santa Fe River","Yes" +"Lou Wilkerson","Yes" +"Lowry, T.S., et al. (2018) Water Resource Assessment in the NM Permian Basin, Sandia Report SAND2018-12018","Yes" +"M.J. Darr consulting letter to Roy Cunnyngham, 2006.","Yes" +"M.J. Darr consulting letter to Roy Cunnyngham, 2006. References additional well tests by M.J. Darr.","Yes" +"Mark Defibaugh","Yes" +"Martinez Surveying Co., [Hydrogeologic Report for] Miller Subdivision, Jan 1984","Yes" +"McCoy, Annie M. and Peery, Roger, 2008, Water-Availability Assessment for Miranda Canyon Preserve Subdivision, Taos County, NM, John Shomaker & Associates. Full report in file called ""Reports w/multi","Yes" +"McCoy, Annie M. and Peery, Roger, 2008, Water-Availability Assessment for Miranda Canyon Preserve Subdivision, Taos County, NM, John Shomaker & Associates. Full report in file called ""Reports w/multip","Yes" +"McLean, J.S. (1970) Saline Ground-Water Resources of the Tularosa Basin, NM. Office of Saline Water, R&D Progress Rpt 561","Yes" +"Meyers et al, 1994, USGS WRI 94-4125","Yes" +"Meyers et al, 1994, USGS WRI 94-4125 (1980 water level)","Yes" +"MJ Darr (July 2006) Pump Test of WL-0007.","Yes" +"Mourant; GWSI; field check by Johnson; USGS well schedules","Yes" +"Mourant; GWSI; field check by Johnson; well log.","Yes" +"MS thesis by Sigstedt, S. (2010) Environmental Tracers in Groundwater of the Salt Basin, NM and Implications for Water Resources","Yes" +"Myers, Everheart, Wilson (1994) Geohydrol. of San Agustin Basin, Alamosa Creek Basin upstream from Monticello Box & Upper Gila Basin in parts of Catron, Socorro & Sierra Cty, NM, USGS WRI 94-4125","Yes" +"Myers, Everheart, Wilson (1994) Geohydrol. of San Agustin Basin, Alamosa Creek Basin upstream from Monticello Box & Upper Gila Basin in parts of Catron/Socorro/Sierra Cty, NM, USGS WRI 94-4125","Yes" +"NA03200901BBB","Yes" +"NA03200912ACC","Yes" +"NA03201066BBB","Yes" +"NA03300814CCC","Yes" +"NA03300914CCC","Yes" +"NA03300921CCC","Yes" +"NA03300922ACC","Yes" +"NA03300933CAB","Yes" +"NA03300933CAB2","Yes" +"NA03300935DBB","Yes" +"NA03300936DBB","Yes" +"NA03301014DDD","Yes" +"NA03301019CCD","Yes" +"NA03301020CCB","Yes" +"NMBG","Yes" +"NMBGMR","Yes" +"NMBGMR & USGS","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000). Shomaker, J.W., 1998, Sustainable Ground-Water Production from City Well Field, Santa Fe, NM, 4/98; Camp Dresser & McKee, Jan. 2000,","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); OSE 2005 WL field sheets; WATERS","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS. OSE Well Record.","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; J. Corbin, written communication, 2007","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; K. Summers project files; JSAI, 2001,App. #6","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; K.Summers project files; JSAI (2001) App. #6","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; K.Summers project files; Minton log and pump test; JS&A, 2001 well log and water levels","Yes" +"NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; Summer's project files; Minton log; Frost, 1996, Review of El Dorado area water production and GW resource, June '96; JSAI, 2001, App. #6","Yes" +"NMED Solid Waste Bureau monitoring reports.","Yes" +"NMED Spring Survey and Monitor Well Survey","Yes" +"NMED SW Bureau data report, URS Corp.; written communications from NMED SolidWaste Bureau","Yes" +"NMED SWP Atlas online","Yes" +"NMOSE POD only","Yes" +"NMOSE Well log","Yes" +"No well inventory form for this site. Just coordinates, photo and well log.","Yes" +"NRCS Well development document (Excel spreadsheet)","Yes" +"OCD AP-100 report by Conestoga-Rovers & Associates","Yes" +"OCD AP-101 Report by Arcadis U.S. Inc.","Yes" +"OCD AP-104 Report by Conestoga-Rovers and Associates","Yes" +"OCD AP-105 Report by GHD","Yes" +"OCD AP-107 Report by Arcadis U.S. Inc","Yes" +"OCD AP-112 Report by Larson & Associates, Inc. ","Yes" +"OCD AP-114 Report by Tasman Geosciences","Yes" +"OCD AP-115 Report by Conestoga-Rovers and Associates","Yes" +"OCD AP-118 Report by GHD","Yes" +"OCD AP-120 Report by GHD Services Inc.","Yes" +"OCD AP-56 Report by GHD","Yes" +"OCD AP-62 Report by R.T. Hicks Consultants, Ltd","Yes" +"OCD AP-71 Report by Rice Operating Company","Yes" +"OCD AP-73 Report by Enviro Clean Cardinal LLC","Yes" +"OCD AP-75 Report by Rice Environmental Consulting & Safety, LLC","Yes" +"OCD AP-87 Report by by Tetra Tech","Yes" +"OCD AP-88 Report by Tetra Tech","Yes" +"OCD AP-91 Report by Talon/LPE","Yes" +"OCD AP-94 Report by Tetra Tech","Yes" +"OCD AP-95 report by Tetra Tech","Yes" +"OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"OFR-51: Hydrogeology of the San Agustin Plains, NM (1973) Blodgett, Daniel D. and Titus, Frank B., NMBGMR","Yes" +"Onwer","Yes" +"Original data and chemistry from Lynn Brandvold with NMBGMR chemistry lab. We visited site in July 2015 took GPS location and found well record.","Yes" +"OSE","Yes" +"OSE and Foreman.","Yes" +"OSE database. Lauren Sherson established USGS Site ID for the NGWMN project (5/29/18 - KP).","Yes" +"OSE field notes","Yes" +"OSE field sheet","Yes" +"OSE macth doubtful, usgs depth is much different, 311 vs 375","Yes" +"OSE POD record","Yes" +"OSE POD record.","Yes" +"OSE Record and owner recollection","Yes" +"OSE records","Yes" +"OSE records and owner recollection.","Yes" +"OSE records and well owner records.","Yes" +"OSE records online.","Yes" +"OSE Records, some info from well owner","Yes" +"OSE Records, well owner provided info.","Yes" +"OSE Records, well owner recollection.","Yes" +"OSE Records.","Yes" +"OSE Records. Gary Goss at TWSD.","Yes" +"OSE records. PN031 may not be it. Well is very old.","Yes" +"OSE WATERS site","Yes" +"OSE website","Yes" +"OSE website info, no well record. Meyers et al, 1994, USGS WRI 94-4125","Yes" +"OSE website, no well record.","Yes" +"OSE website, no well record. Meyers et al, 1994, USGS WRI 94-4125","Yes" +"OSE website, well record available at OSE office.","Yes" +"OSE website, well record maybe available at OSE office.","Yes" +"OSE Well Log and PNM Public Water System Information Sheet; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; C. Borchert, written communication, 3/23/09","Yes" +"OSE Well Log and PNM Public Water System Information Sheet; NMED DWB (1999) via Jemez y Sangre Database (2000); Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; C.Borchert, 3/23/09","Yes" +"OSE well log. Some data from USGS. Overlapping site data.","Yes" +"OSE Well record","Yes" +"OSE well record and owner","Yes" +"OSE well record and owner info","Yes" +"OSE well record and site visit","Yes" +"OSE Well Record and well owner","Yes" +"OSE well record.","Yes" +"OSE Well Record. NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS","Yes" +"OSE Well Record. NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS; K. Summers project file and Sandia Drilling well log","Yes" +"OSE Well Record. NMED DWB (1999) via Jemez y Sangre Database (2000); WATERS","Yes" +"Owner","Yes" +"Owner - well dirlled in 60s.","Yes" +"owner - well drilled by father, using septic auger.","Yes" +"Owner - well drilled in 50's. owner has some records.","Yes" +"Owner and OSE record","Yes" +"Owner and OSE site.","Yes" +"Owner and OSE well record.","Yes" +"Owner and well record.","Yes" +"Owner field check; WATERS","Yes" +"Owner recollection","Yes" +"Owner recollection and limited OSE records.","Yes" +"Owner recollection and OSE POD record.","Yes" +"Owner recollection of well depth and water level","Yes" +"Owner recollection.","Yes" +"Owner recollection; OSE Records","Yes" +"Owner.","Yes" +"Owner. OSE well records","Yes" +"P. Drakos, et al., 2004, Chemical & Isotopic Constraints on Source-Waters and Connectivity of Basin-Fill Aquifers in the Southern San Luis Basin, NM. NMGS Guidebook 55, pg 405.","Yes" +"Pecos Valley Artesian Conservancy District (Aron Balok okay'd us to share data from PVACD wells to the web map on 3/30/2022 via email with Stacy)",NO +"Pecos Valley Artesian Conservancy District (Aron Balok okay'd us to share data from PVACD wells to the web map on 3/30/2022 via email with Stacy). Same well as USGS 331524104245101.",NO +"Pecos Valley Artesian Conservancy District (Aron Balok okay'd us to share data from PVACD wells to the web map on 3/30/2022 via email with Stacy)",NO +"PhD thesis by Mayer, J.R. (1995) Role of Fractures in Regional GW Flow: Field Evidence and Model Results from the Basin/Range of TX and NM","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID # S264)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S037)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S040)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S101)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S146)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S164)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S220)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131 (ID #S265)","Yes" +"Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131.","Yes" +"Plummer, LN, Bexfield, LM, et al., 2004 Geochem. Characterization of Grd-water Flow in the Santa Fe Grp Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131.","Yes" +"Pre-design groundwater investigation at the tailing facility - Final work plan. April 24, 2013, Arcadis","Yes" +"Pretty certain C011 is Burns EPA sample.","Yes" +"Prindle-Hinds well completion log to NMED SW Bureau","Yes" +"Pump test performed as noted in Meyers report. USGS WRI 94-4125","Yes" +"Ranch foreman.","Yes" +"Rankin, Dale, 2005, Personal Communication USGS Well Drilling Reports","Yes" +"Reardon et al. (2021) Addendum to Water Res. Assessment NM Permian Basin SAND2021-1869","Yes" +"Reardon, et al. (2021) Addendum to Water Res. Assessment in NM Permian Basin. SAND2021-1869.","Yes" +"RGDSSP12","Yes" +"Romero","Yes" +"San Pedro Creek Estates HOA","Yes" +"Sandia Drilling well log from JSA, 2001, App. 6; Frost & Assoc., 1996, Review of El Dorado Area Water Production and GW Resource, June 1996; WKSummers files","Yes" +"Sandia Drilling well log from JSA, 2001, App. 6; Frost & Assoc., 1996, Review of El Dorado Area Water Production and GW Resource, June 1996; WKSummers files; DKoning cutting review","Yes" +"Sandia National Labs/ BLM Wells","Yes" +"Sandia Natl. Labs Report SAND2018-12018 Water Assessment in the NM Permian Basin","Yes" +"Sangre de Cristo/OSE, WATERS, Borchert report, Watson and Rappuhn/Shomaker & Assoc., 4/99, WellReport: Drilling, Construction and Testing, City of Santa Fe Northwest Area Test Well; D.Koning review","Yes" +"Santa Fe County records; well log; field check","Yes" +"Santa Fe County Utilities field notes;","Yes" +"SECOR report to NMED UST Bureau","Yes" +"Shoemaker & Fleming, MWB Survey, 1990; Mourant, 1980","Yes" +"Shomaker & Associates, 2004, Well Report Buckman Wells No. 10-13 City of Santa Fe, New Mexico, May 2004; C. Borchert, written communication, 3/23/09","Yes" +"Shomaker & Associates, April 2003, Well Report: Drilling, Construction, and Testing of SF Buckman 9; C. Borchert, written communication, 3/23/09","Yes" +"Site check; well record","Yes" +"Site visit by Johnson during drilling; drillers report; J. Corbin, written communication, 2007","Yes" +"Slinglerland & Borton, Geohydro of Avanti Business Park, 8/85 (App F to GGI, Santa Fe Metro, 12/01); lithologic log by GGI, review by D.Koning","Yes" +"SNL report for manual water level and water quality","Yes" +"Some chemistry from EPA via NM Health Dept.","Yes" +"Some data from USGS. Overlapping site data.","Yes" +"Some data from USGS. Overlapping site data. Aquifer type from USGS.","Yes" +"Some data from USGS. Overlapping site data. Formation pick from USGS.","Yes" +"Some limited WQ measurements at well and nearby stream. Filed. Not entered.","Yes" +"Some water levels from USGS","Yes" +"Some WL's from Lee Foster Well Services","Yes" +"Some WLs from USGS & Dames and Moore (4/11/1986). Some WQ from Vail Engineering (Sept. 1993).","Yes" +"Souder-Miller & Assoc., 1995, Unpublished Consultant's Report, Fig 2, well completion diagram","Yes" +"Source data for some chemistry for this location is from the US EPA.","Yes" +"Source data for this location is from the US EPA & NMED water quality.","Yes" +"Source data for this location is from the US EPA.","Yes" +"T.Decker, written comm., 2000 (letter to B.McLean ""Testing - EUI Well #15"", 2/25/2000);","Yes" +"Taos Soil & Water Concervation District","Yes" +"Taos Soil & Water Conservation District","Yes" +"Taos Soil & Water Conservation District water sample study.","Yes" +"Tetra Tech EM, Inc., Aug 2004, Geohydrologic Report for Proposed Suerte del Sur Subdivision, Santa Fe County, NM","Yes" +"Tetra Tech EM, Inc., Aug 2004, Geohydrologic Report for Proposed Suerte del Sur Subdivision, Santa Fe County, NM; field check by Johnson","Yes" +"Texas Water Development Board",NO +"Texas Water Development Board; Kreitler, et al. (1987) Siting a low-level radioactive waste disposal facility in TX, Vol. 4. (pump test data)",NO +"Theis, C.V., Taylor, Jr., G.C., and Murray, C.R. (1941) Thermal Waters of the Hot Springs Artesian Basin Sierra County, New Mexico. U.S. Geological Survey in cooperation with State Engineer of New Mex","Yes" +"TSWCD","Yes" +"Turner, B., 1996, The Water Availability for the El Mirador Subdivision, Taos County, New Mexico, Turner Environmental Consultants, Apdx. 2 & 3. (Data from Drakos, GGI, 1996) Report filed w/TV-229. Al","Yes" +"Turner, B., 1996, The Water Availability for the El Mirador Subdivision, Taos County, New Mexico, Turner Environmental Consultants, Appendices 1, 3, 4","Yes" +"Turner, B., 1996, The Water Availability for the El Mirador Subdivision, Taos County, New Mexico, Turner Environmental Consultants, pg. 7, Table 1.","Yes" +"Turner, B., 1996, The Water Availability for the El Mirador Subdivison, Taos County, New Mexico, Turner Environmental Consultants, Pg. 18, Table 1, and Appendices 2,3. (Data from Jenkins in 1982). Ful","Yes" +"undeclared well.","Yes" +"URS Corp. for Chevron","Yes" +"USFS - Espanola Ranger District - Range Improvement records.","Yes" +"USGS",NO +"USGS & Garrabrant, L.A. (1993) Water Resources of Taos County, NM USGS WRIR 93-4107 (chemistry sample). This well matched to Garrabrant well via notes from Tony Benson's Sunshine notebooks.","Yes" +"USGS & Myers, Everheart, Wilson (1994) Geohydrol. Of San Agustin Basin, Alamosa Creek Basin upstream from Monticello Box & Upper Gila Basin in parts of Catron, Socorro & Sierra Cty, NM, WRI 94-4125","Yes" +"USGS & NMED SWP Atlas online","Yes" +"USGS & Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131","Yes" +"USGS & Texas Water Development Board","Yes" +"USGS & Winograd, I.J. (1959) Ground-water Conditions & Geology of Sunshine Valley & Western Taos Cty, NM, NMSEO, Tech. Report #12. & Garrabrant, L.A. (1993) USGS WRIR 93-4107","Yes" +"USGS & Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12","Yes" +"USGS & Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12.","Yes" +"USGS and Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12.","Yes" +"USGS Chem Analysis Sheet from R.L. Borton Files. 1970; WATERS; C.Borchert, 3/23/09, written communication","Yes" +"USGS Drillers reports","Yes" +"USGS drillers reports; D.Koning cuttings review","Yes" +"USGS drilling reports; D.Koning cutting review","Yes" +"USGS NWIS data site",NO +"USGS NWIS data siteSome data from USGS. Overlapping site data.",NO +"USGS Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12","Yes" +"USGS WLs",NO +"USGS, chemistry from Lynn Brandvold with NMBGMR chemistry lab, OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"USGS. POD log not available.","Yes" +"USGS. Well screens/WQ from Plummer, LN, Bexfield, LM, et al. (2004) Geochem. Characterization of Ground-water Flow in the Santa Fe Group Aquifer System, Middle Rio Grande Basin, NM. USGS WRI-03-4131","Yes" +"USGS/OSE water-level data sheets; WATERS","Yes" +"USGS; Bjorklund, L.J. (1957) Recon. of Groundwater Conditions in Crow Flats Area Otero Cty, NM, OSE Tech Rep 8.","Yes" +"USGS; Bjorklund, L.J. (1957) Reconnaissance of Ground-water Conditions in the Crow Flats Area Otero County, New Mexico, NM OSE Technical Report 8.","Yes" +"USGS; Bjorklund, L.J. (1957) Reconnaissance of Ground-water Conditions in the Crow Flats Area Otero County, NM, OSE Technical Report 8.","Yes" +"USGS; Screens from NPS water level and well documentation","Yes" +"VeneKlasen & Assoc, 1984, Remuda Ridge Warehouse, Santa Fe County, NM, October 1984","Yes" +"VeneKlasen & Assoc, Inc., 1982, Geohydrology Rept, Pecos Trail Subdn, Santa Fe Cnty, Aug 1982","Yes" +"VeneKlasen & Assoc., 1984, Geohydrology Report, Hondo Trails Subdn, Santa Fe Cnty, Dec 1984","Yes" +"VeneKlasen & Assoc., Inc., 1986, Geohydrology Report, Talbot Subdivision, May 1986","Yes" +"VeneKlasen, 1975, Hydrogeologic Report for La Tierra Subdivision, 6/75","Yes" +"Veneklasen, 1977, Geohydro Lanphere -- Rio Villa Subdivision, AguaFria Rd, 5/77","Yes" +"Veneklasen, 1977, Hydrogeologic Report La Tierra Subdivn, Phase III, May 1977, Consulting Prof Inc","Yes" +"VeneKlasen, 1977, Hydrogeologic Rept La Tierra Ph. III, May 1977","Yes" +"VeneKlasen, 1977, Hydrogeologic Rept La Tierra Ph. III, May 1977; field check Cruz/Frost","Yes" +"Veneklasen, 1980, Geohydro of Santiago Subdivision, 10/80","Yes" +"VeneKlasen, 6/75, Hydrogeo Rept La Tierra; field check by Johnson","Yes" +"VeneKlasen, Hydrogeologic Report for La Tierra Subdivision 6/75; well log; field check Johnson","Yes" +"Water chemistry from TSWCD. Fairly sure this is COS-11, eventhough there is a descrepency between well log depth and well depth on chemistry sheet.","Yes" +"Water Quality: Sorrell, John D. and Banet, Christopher, Karavas Tract Exploratory Well, Nov. 1993, Bureau of Indian Affairs.","Yes" +"WATERS; field check","Yes" +"WATERS; Lewis database, 2001; VeneKlasen, 1975, Hydrogeologic Report LaTierra, 6/75; Camp Dresser & McKee, Inc., Jan 2001, Water Supply Analysis; D.Koning log review; C.Borchert, 3/23/09, written comm","Yes" +"WATERS; WK Summers project files; JSAI, 2001, App. #6; J. Frost project files (1999)","Yes" +"Watson/Shomaker & Assoc., 5/97, Well Report, Drilling, Construction, and Testing, City of Santa Fe, Torreon Well No. 2, for PNM Water Services Santa Fe, NM; CDM, Jan 2000, Water Supply Analysis","Yes" +"Well owner.","Yes" +"Well also in PhD thesis by Mayer, J.R. (1995) Role of Fractures in Regional GW Flow: Field Evidence and Model Results from the Basin/Range of TX and NM","Yes" +"well depth from Meyers et al, 1994, USGS WRI 94-4125","Yes" +"Well in Meyers report: Meyers et al, 1994, USGS WRI 94-4125","Yes" +"Well info from inside well cap, presumably from driller?","Yes" +"Well Log","Yes" +"Well log and completion diagrams","Yes" +"Well log and field check by Johnson; not previously in GWSI","Yes" +"Well log and well completion diagrams","Yes" +"Well log and well completion diagrams. WQ data from P. Longmire Santa Fe River 1985 Report","Yes" +"Well log/record by GGI; field check by Johnson","Yes" +"Well log; field check","Yes" +"Well log; field check by Johnson","Yes" +"well log; field check by Johnson; owner","Yes" +"well log; Johnson field check","Yes" +"Well Log; JSAI, 2001, App. #6","Yes" +"Well owner","Yes" +"Well owner and old well record.","Yes" +"Well owner and OSE site info.","Yes" +"Well owner, and well record.","Yes" +"Well owner.","Yes" +"Well record.","Yes" +"Well record; field check","Yes" +"Well record; field check by Cruz/Frost","Yes" +"well record; field check by Johnson","Yes" +"Well record; field check by Johnson; owner","Yes" +"Well record; field check by Lyman","Yes" +"Well record; field check by Lyman; SWL measured by GGI/reported by Cooper, 2000, Rept for Roybal","Yes" +"Well record; field check with SFC Utilities (Leonard Quintana 490-0065)","Yes" +"Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12","Yes" +"Winograd, I.J., 1959, Ground-water Conditions and Geology of Sunshine Valley and Western Taos County, NM, State of New Mexico, State Engineer Office, Technical Report #12.","Yes" +"WK Summers project files; WATERS; Water Industries, Inc., 1982, Testing of Eldorado Well #1, Jan. 1982; water levels from W-1 and EUI#1; JSAI, 2001, App. #6","Yes" +"WL from USGS; chemistry and other data from Lynn Brandvold with NMBGMR chemistry lab & OFR-469 Hydrogeology & Water Resources of the Placitas Area","Yes" +"WLs from USGS",NO +"WQ also from MS thesis by Sigstedt, S. (2010) Environmental Tracers in Groundwater of the Salt Basin, NM and Implications for Water Resources.","Yes" +"WQ from Taos Soil & Water Conservation District","Yes" +"Z. Spiegel, 1972, Interpretation and application of an aquifer performance test on well RG-20228 at Pojoaque Terrace trailer court site HC McDonald Property, Santa Fe County, NM, 9/14/72","Yes" diff --git a/transfers/transfer.py b/transfers/transfer.py index 1c094753e..cf409e3ce 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -74,6 +74,7 @@ def message(msg, pad=10, new_line_at_top=True): logger.info(f"{pad} {msg} {pad}") +@timeit def transfer_all(sess, limit=100): message("STARTING TRANSFER", new_line_at_top=False) erase_and_initalize(sess) @@ -115,15 +116,16 @@ def transfer_all(sess, limit=100): # cleanup_wells(sess) -@timeit -def main_transfer(): +def main(): limit = int(os.environ.get("TRANSFER_LIMIT", 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) + #todo: move the log file to a storage bucket + if __name__ == "__main__": - main_transfer() + main() # ============= EOF ============================================= diff --git a/transfers/util.py b/transfers/util.py index 5af2777ae..293894a48 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import csv from datetime import datetime, timezone, timedelta import pytz import re @@ -116,6 +117,25 @@ def extract_organization(alternate_id: str) -> str: return "Unknown" +def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: + with open('data/valid_welldata_datasources.csv', 'r') as f: + reader = csv.reader(f) + _ = next(reader) + valid_datasources = [row[0] for row in reader if row[1]=='Yes'] + logger.info(f'Valid WellData Datasources: {valid_datasources}') + + return df[df["DataSource"].isin(valid_datasources)] + + +def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: + with open('data/valid_measuring_agency.csv', 'r') as f: + reader = csv.reader(f) + _ = next(reader) + valid_measuring_agencies = [row[0] for row in reader if row[1]=='Yes'] + logger.info(f'Valid Measuring Agencies: {valid_measuring_agencies}') + return df[df["MeasuringAgency"].isin(valid_measuring_agencies)] + + def filter_to_valid_point_ids(session: Session, df: pd.DataFrame) -> pd.DataFrame: valid_point_ids = get_valid_point_ids(session) return df[df["PointID"].isin(valid_point_ids)] diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 563372f6a..b202496eb 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -24,7 +24,7 @@ filter_to_valid_point_ids, logger, read_csv, - convert_mt_to_utc, + convert_mt_to_utc, filter_by_valid_measuring_agency, ) @@ -32,6 +32,7 @@ def transfer_water_levels(session): wd = read_csv("WaterLevels") wd = filter_to_valid_point_ids(session, wd) + wd = filter_by_valid_measuring_agency(wd) gwd = wd.groupby(["PointID"]) start_time = time.time() diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index bd70cb00c..3c4d2f17a 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -30,7 +30,7 @@ filter_to_valid_point_ids, read_csv, logger, - replace_nans, + replace_nans, filter_by_welldata_datasource, ) ADDED = [] @@ -46,13 +46,13 @@ def transfer_wells(session, limit=0): wdf = replace_nans(wdf) + #todo: filter Locations by DataSource + wdf = filter_by_welldata_datasource(wdf) + n = len(wdf) - start_time = time.time() - results = { - "n": n, - } - made_things = [] + for i, row in enumerate(wdf.itertuples()): + start_time = time.time() pointid = row.PointID if wdf[wdf["PointID"] == pointid].shape[0] > 1: logger.warning(f"PointID {pointid} has duplicate records. Skipping.") @@ -114,11 +114,8 @@ def transfer_wells(session, limit=0): assoc.location = location assoc.thing = well session.add(assoc) - made_things.append(row.PointID) - results["made_things"] = made_things session.commit() - return results def transfer_wellscreens(session, limit=None): From 624756314eee7f8a5f982bbbd9b92609e8c8e26d Mon Sep 17 00:00:00 2001 From: jirhiker Date: Wed, 24 Sep 2025 01:41:01 +0000 Subject: [PATCH 19/47] Formatting changes --- transfers/transfer.py | 2 +- transfers/util.py | 12 ++++++------ transfers/waterlevels_transfer.py | 3 ++- transfers/well_transfer.py | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index cf409e3ce..ab070e778 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -122,7 +122,7 @@ def main(): with session_ctx() as sess: transfer_all(sess, limit=limit) - #todo: move the log file to a storage bucket + # todo: move the log file to a storage bucket if __name__ == "__main__": diff --git a/transfers/util.py b/transfers/util.py index 293894a48..96dec3db1 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -118,21 +118,21 @@ def extract_organization(alternate_id: str) -> str: def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: - with open('data/valid_welldata_datasources.csv', 'r') as f: + with open("data/valid_welldata_datasources.csv", "r") as f: reader = csv.reader(f) _ = next(reader) - valid_datasources = [row[0] for row in reader if row[1]=='Yes'] - logger.info(f'Valid WellData Datasources: {valid_datasources}') + valid_datasources = [row[0] for row in reader if row[1] == "Yes"] + logger.info(f"Valid WellData Datasources: {valid_datasources}") return df[df["DataSource"].isin(valid_datasources)] def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: - with open('data/valid_measuring_agency.csv', 'r') as f: + with open("data/valid_measuring_agency.csv", "r") as f: reader = csv.reader(f) _ = next(reader) - valid_measuring_agencies = [row[0] for row in reader if row[1]=='Yes'] - logger.info(f'Valid Measuring Agencies: {valid_measuring_agencies}') + valid_measuring_agencies = [row[0] for row in reader if row[1] == "Yes"] + logger.info(f"Valid Measuring Agencies: {valid_measuring_agencies}") return df[df["MeasuringAgency"].isin(valid_measuring_agencies)] diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index b202496eb..fcf6376bc 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -24,7 +24,8 @@ filter_to_valid_point_ids, logger, read_csv, - convert_mt_to_utc, filter_by_valid_measuring_agency, + convert_mt_to_utc, + filter_by_valid_measuring_agency, ) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 3c4d2f17a..8d1d54250 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -30,7 +30,8 @@ filter_to_valid_point_ids, read_csv, logger, - replace_nans, filter_by_welldata_datasource, + replace_nans, + filter_by_welldata_datasource, ) ADDED = [] @@ -46,7 +47,7 @@ def transfer_wells(session, limit=0): wdf = replace_nans(wdf) - #todo: filter Locations by DataSource + # todo: filter Locations by DataSource wdf = filter_by_welldata_datasource(wdf) n = len(wdf) From d0b4a43b741cd953eacc831a560862f55b6f3bd1 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 23 Sep 2025 19:49:38 -0600 Subject: [PATCH 20/47] feat: update file paths for valid well data sources and measuring agencies --- transfers/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index 96dec3db1..ff24ded9b 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -118,7 +118,7 @@ def extract_organization(alternate_id: str) -> str: def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: - with open("data/valid_welldata_datasources.csv", "r") as f: + with open("/workspace/transfers/data/valid_welldata_datasources.csv", "r") as f: reader = csv.reader(f) _ = next(reader) valid_datasources = [row[0] for row in reader if row[1] == "Yes"] @@ -128,7 +128,7 @@ def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: - with open("data/valid_measuring_agency.csv", "r") as f: + with open("/workspace/transfers/data/valid_measuring_agency.csv", "r") as f: reader = csv.reader(f) _ = next(reader) valid_measuring_agencies = [row[0] for row in reader if row[1] == "Yes"] From ec0143259c89e0a555a44960920587c29c9ef0dc Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 24 Sep 2025 08:45:06 -0600 Subject: [PATCH 21/47] feat: enhance data transfer process with schema management and file path validation --- db/location.py | 2 +- transfers/transfer.py | 5 ++++ transfers/util.py | 13 ++++++++-- transfers/waterlevels_transfer.py | 12 +++++----- transfers/well_transfer.py | 40 +++++++++++++++++++++---------- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/db/location.py b/db/location.py index 7cccd2db0..20a854446 100644 --- a/db/location.py +++ b/db/location.py @@ -39,7 +39,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} nma_pk_location: Mapped[UUID] = mapped_column( - String(36), nullable=True, unique=True + String(36), nullable=True ) description: Mapped[str] = mapped_column # name: Mapped[str] = mapped_column(String(255), nullable=True) diff --git a/transfers/transfer.py b/transfers/transfer.py index fadbfe681..0b4176b8c 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -62,6 +62,11 @@ def lexicon(): @timeit def erase(session: Session): logger.info("Erasing existing data") + from sqlalchemy import text + with session.bind.connect() as conn: + conn.execute(text("DROP SCHEMA public CASCADE")) + conn.execute(text("CREATE SCHEMA public")) + Base.metadata.drop_all(session.bind) logger.info("Recreating tables") Base.metadata.create_all(session.bind) diff --git a/transfers/util.py b/transfers/util.py index ff24ded9b..3ab4c7644 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import csv +import os from datetime import datetime, timezone, timedelta import pytz import re @@ -118,7 +119,11 @@ def extract_organization(alternate_id: str) -> str: def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: - with open("/workspace/transfers/data/valid_welldata_datasources.csv", "r") as f: + path = "/workspace/transfers/data/valid_welldata_datasources.csv" + if not os.path.exists(path): + path = "data/valid_welldata_datasources.csv" + + with open(path, "r") as f: reader = csv.reader(f) _ = next(reader) valid_datasources = [row[0] for row in reader if row[1] == "Yes"] @@ -128,7 +133,11 @@ def filter_by_welldata_datasource(df: pd.DataFrame) -> pd.DataFrame: def filter_by_valid_measuring_agency(df: pd.DataFrame) -> pd.DataFrame: - with open("/workspace/transfers/data/valid_measuring_agency.csv", "r") as f: + path = "/workspace/transfers/data/valid_measuring_agency.csv" + if not os.path.exists(path): + path = "data/valid_measuring_agency.csv" + + with open(path, "r") as f: reader = csv.reader(f) _ = next(reader) valid_measuring_agencies = [row[0] for row in reader if row[1] == "Yes"] diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 6688297ab..408b13c66 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -82,10 +82,10 @@ def transfer_water_levels(session): measurement is the same as the date/time of the field event. """ - if pd.isna(row.MeasuringAgency): - collecting_organization = "Unknown" - else: - collecting_organization = row.MeasuringAgency + # if pd.isna(row.MeasuringAgency): + # collecting_organization = "Unknown" + # else: + # collecting_organization = row.MeasuringAgency if pd.isna(row.MeasuredBy): sampler_name = "Unknown" @@ -95,7 +95,7 @@ def transfer_water_levels(session): field_event = FieldEvent( thing=thing, event_date=dt_utc, - collecting_organization=collecting_organization, + # collecting_organization=collecting_organization, release_status=release_status, ) @@ -117,7 +117,7 @@ def transfer_water_levels(session): sample = Sample( field_activity=field_activity, - sampler_name=sampler_name, + # sampler_name=sampler_name, sample_date=dt_utc, sample_matrix="water", sample_name=str( diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 8d1d54250..e07de958d 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -18,7 +18,7 @@ from sqlalchemy import select from db import LocationThingAssociation, Thing, WellScreen, Location -from schemas.thing import CreateWellScreen +from schemas.thing import CreateWellScreen, CreateWell from services.thing_helper import add_thing from services.util import ( get_state_from_point, @@ -52,8 +52,8 @@ def transfer_wells(session, limit=0): n = len(wdf) + start_time = time.time() for i, row in enumerate(wdf.itertuples()): - start_time = time.time() pointid = row.PointID if wdf[wdf["PointID"] == pointid].shape[0] > 1: logger.warning(f"PointID {pointid} has duplicate records. Skipping.") @@ -67,7 +67,7 @@ def transfer_wells(session, limit=0): logger.info( f"Processing row {i} of {n}. {row.PointID}, avg rows per second: {i / (time.time() - start_time):.2f}" ) - session.commit() + start_time = time.time() try: location = make_location(row) @@ -79,23 +79,32 @@ def transfer_wells(session, limit=0): session.add(location) # TODO: add guards for null values - well = add_thing( - session, - { + # TODO: use schema to validate + + data = CreateWell( # "nma_pk_welldata": row.WellID, - "name": row.PointID, - "hole_depth": row.HoleDepth, - "well_depth": row.WellDepth, + name=row.PointID, + hole_depth= row.HoleDepth, + well_depth= row.WellDepth, # "driller_name": row.DrillerName, # "construction_method": row.ConstructionMethod, # "casing_diameter": row.CasingDiameter, # "casing_depth": row.CasingDepth, # "casing_description": row.CasingDescription, - "release_status": "public" if row.PublicRelease else "private", + release_status= "public" if row.PublicRelease else "private", # "data_reliability": row.DataReliability, - }, - thing_type="water well", + ) + try: + well = add_thing( + session, + data, + thing_type="water well", + ) + except Exception as e: + session.rollback() + logger.warning(f"Error creating well for {row.PointID}: {e}") + continue # TODO: use current use LUT to get well type # wt = row.Meaning @@ -116,7 +125,12 @@ def transfer_wells(session, limit=0): assoc.thing = well session.add(assoc) - session.commit() + try: + session.commit() + except Exception as e: + logger.exception(f"Error committing well {row.PointID}: {e}") + session.rollback() + continue def transfer_wellscreens(session, limit=None): From d68edc59c7c15d05f79551aa238eb021619e8950 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Wed, 24 Sep 2025 14:45:28 +0000 Subject: [PATCH 22/47] Formatting changes --- db/location.py | 4 +--- transfers/transfer.py | 1 + transfers/well_transfer.py | 23 +++++++++++------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/db/location.py b/db/location.py index 20a854446..ad1a99e5d 100644 --- a/db/location.py +++ b/db/location.py @@ -38,9 +38,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): __versioned__ = {} - nma_pk_location: Mapped[UUID] = mapped_column( - String(36), nullable=True - ) + nma_pk_location: Mapped[UUID] = mapped_column(String(36), nullable=True) description: Mapped[str] = mapped_column # name: Mapped[str] = mapped_column(String(255), nullable=True) point: Mapped[WKBElement] = mapped_column( diff --git a/transfers/transfer.py b/transfers/transfer.py index 0b4176b8c..26b3d9c93 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -63,6 +63,7 @@ def lexicon(): def erase(session: Session): logger.info("Erasing existing data") from sqlalchemy import text + with session.bind.connect() as conn: conn.execute(text("DROP SCHEMA public CASCADE")) conn.execute(text("CREATE SCHEMA public")) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index e07de958d..c785d4f5b 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -82,18 +82,17 @@ def transfer_wells(session, limit=0): # TODO: use schema to validate data = CreateWell( - # "nma_pk_welldata": row.WellID, - name=row.PointID, - hole_depth= row.HoleDepth, - well_depth= row.WellDepth, - # "driller_name": row.DrillerName, - # "construction_method": row.ConstructionMethod, - # "casing_diameter": row.CasingDiameter, - # "casing_depth": row.CasingDepth, - # "casing_description": row.CasingDescription, - release_status= "public" if row.PublicRelease else "private", - # "data_reliability": row.DataReliability, - + # "nma_pk_welldata": row.WellID, + name=row.PointID, + hole_depth=row.HoleDepth, + well_depth=row.WellDepth, + # "driller_name": row.DrillerName, + # "construction_method": row.ConstructionMethod, + # "casing_diameter": row.CasingDiameter, + # "casing_depth": row.CasingDepth, + # "casing_description": row.CasingDescription, + release_status="public" if row.PublicRelease else "private", + # "data_reliability": row.DataReliability, ) try: well = add_thing( From 9c614bd942db86ef50099323da8f517dc3ce0df4 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 24 Sep 2025 14:14:05 -0600 Subject: [PATCH 23/47] feat: refactor logging implementation and enhance error handling in transfer scripts --- transfers/asset_transfer.py | 3 +- transfers/contact_transfer.py | 8 ++-- transfers/group_transfer.py | 3 +- transfers/logger.py | 61 +++++++++++++++++++++++++++++++ transfers/thing_transfer.py | 7 ++-- transfers/transfer.py | 5 ++- transfers/util.py | 38 +------------------ transfers/waterlevels_transfer.py | 32 +++++++++------- 8 files changed, 95 insertions(+), 62 deletions(-) create mode 100644 transfers/logger.py diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 3c90d2264..0e2f6cd36 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -32,7 +32,8 @@ get_storage_bucket, get_storage_client, ) -from transfers.util import get_valid_things, logger +from transfers.util import get_valid_things +from transfers.logger import logger def transfer_assets(session: Session) -> None: diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 7d2234f8b..e0a144185 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -15,9 +15,9 @@ # =============================================================================== import numpy as np import pandas as pd -from transfers.util import read_csv, filter_to_valid_point_ids, logger, replace_nans +from transfers.util import read_csv, filter_to_valid_point_ids, replace_nans +from transfers.logger import logger from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address - from schemas.contact import CreateContact, CreateAddress @@ -60,7 +60,7 @@ def transfer_contacts(session): thing = session.query(Thing).where(Thing.name == row.PointID).first() logger.info(f"Processing PointID: {i} {row.PointID}") if thing is None: - logger.warning( + logger.critical( f"Thing with PointID {row.PointID} not found. Skipping owner." ) continue @@ -178,7 +178,7 @@ def add_first_contact(session, row, thing): CreateAddress.model_validate(address_data) contact.addresses.append(Address(**address_data)) except Exception as e: - logger.warning( + logger.critical( f"Skipping physical address for first contact {name}. Validation error: {e}" ) diff --git a/transfers/group_transfer.py b/transfers/group_transfer.py index 12bcf452f..95e9c7949 100644 --- a/transfers/group_transfer.py +++ b/transfers/group_transfer.py @@ -18,7 +18,8 @@ from db import Thing, Group from db.engine import session_ctx -from transfers.util import read_csv, logger +from transfers.util import read_csv +from transfers.logger import logger def transfer_groups( diff --git a/transfers/logger.py b/transfers/logger.py new file mode 100644 index 000000000..5c08fc2c2 --- /dev/null +++ b/transfers/logger.py @@ -0,0 +1,61 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import logging +import sys +from datetime import datetime + +from services.gcs_helper import get_storage_bucket + + +# class StreamToLogger: +# def __init__(self, logger_, level): +# self.logger = logger_ +# self.level = level +# self.linebuf = "" +# +# def write(self, buf): +# for line in buf.rstrip().splitlines(): +# self.logger.log(self.level, line.rstrip()) +# +# def flush(self): +# pass + + +log_filename = f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)-8s] %(message)s", + handlers=[ + logging.StreamHandler(), + logging.FileHandler(log_filename, mode="w", encoding="utf-8"), + ], +) +logger = logging.getLogger(__name__) + +# workaround to not redirect httpx logging +logging.getLogger("httpx").setLevel(logging.WARNING) + +# redirect stderr to the logger +# sys.stderr = StreamToLogger(logger, logging.ERROR) + +def save_log_to_bucket(): + bucket = get_storage_bucket() + blob = bucket.blob(f'transfer_logs/{log_filename}') + blob.upload_from_filename(log_filename) + logger.info(f"Uploaded log to gs://{bucket.name}/transfer_logs/{log_filename}") +# ============= EOF ============================================= diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 39a688cf7..2172575fd 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -18,7 +18,8 @@ from db import LocationThingAssociation from services.thing_helper import add_thing -from transfers.util import make_location, read_csv, logger, replace_nans +from transfers.util import make_location, read_csv, replace_nans +from transfers.logger import logger def transfer_thing(session: Session, site_type: str, make_payload, limit=None) -> None: @@ -32,7 +33,7 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - for i, row in enumerate(ldf.itertuples()): pointid = row.PointID if ldf[ldf["PointID"] == pointid].shape[0] > 1: - logger.warning(f"PointID {pointid} has duplicate records. Skipping.") + logger.critical(f"PointID {pointid} has duplicate records. Skipping.") continue if limit and i >= limit: @@ -48,7 +49,7 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - try: location = make_location(row) except Exception as e: - logger.error(f"Error creating location for {row.PointID}: {e}") + logger.critical(f"Error creating location for {row.PointID}: {e}") continue session.add(location) payload = make_payload(row) diff --git a/transfers/transfer.py b/transfers/transfer.py index 26b3d9c93..40f9a67cd 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -37,7 +37,8 @@ transfer_ephemeral_stream, transfer_met, ) -from transfers.util import logger, timeit, timeit_direct +from transfers.util import timeit, timeit_direct +from transfers.logger import logger, save_log_to_bucket def erase_and_initalize(session: Session) -> None: @@ -139,7 +140,7 @@ def main(): transfer_all(sess, limit=limit) # todo: move the log file to a storage bucket - + save_log_to_bucket() if __name__ == "__main__": main() diff --git a/transfers/util.py b/transfers/util.py index 3ab4c7644..6d9d5d544 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -19,7 +19,6 @@ import pytz import re import io -import logging from shapely import Point @@ -37,42 +36,7 @@ get_county_from_point, get_quad_name_from_point, ) -import sys - - -class StreamToLogger: - def __init__(self, logger, level): - self.logger = logger - self.level = level - self.linebuf = "" - - def write(self, buf): - for line in buf.rstrip().splitlines(): - self.logger.log(self.level, line.rstrip()) - - def flush(self): - pass - - -# todo: setup of logging should be moved to function -log_filename = f"transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" - - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)-8s] %(message)s", - handlers=[ - logging.StreamHandler(), - logging.FileHandler(log_filename, mode="w", encoding="utf-8"), - ], -) -logger = logging.getLogger(__name__) - -# workaround to not redirect httpx logging -logging.getLogger("httpx").setLevel(logging.WARNING) - -# redirect stderr to the logger -sys.stderr = StreamToLogger(logger, logging.ERROR) +from logger import logger def replace_nans(df: pd.DataFrame, default=None) -> pd.DataFrame: diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 408b13c66..b1f6a4c1d 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -39,7 +39,15 @@ def transfer_water_levels(session): start_time = time.time() for index, group in gwd: - logger.info(f"Processing PointID: {index[0]}") + pointid = index[0] + logger.info(f"Processing PointID: {pointid}") + thing = session.query(Thing).where(Thing.name == pointid).first() + if thing is None: + logger.critical( + f"Thing with PointID={pointid} not found. Skipping water levels" + ) + continue + n = len(group) for i, row in enumerate(group.itertuples()): if i and not i % 25: @@ -49,7 +57,8 @@ def transfer_water_levels(session): session.commit() if pd.isna(row.DepthToWater) or pd.isna(row.DateMeasured): - logger.warning(f"Skipping row {row.Index} due to missing data.") + logger.critical(f"Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to missing " + f"data.") continue if not pd.isna(row.TimeMeasured): @@ -62,14 +71,7 @@ def transfer_water_levels(session): dt_utc = convert_mt_to_utc(dt) except ValueError as e: logger.warning( - f"Skipping row {row.Index} due to invalid date/time: {e}" - ) - continue - - thing = session.query(Thing).where(Thing.name == row.PointID).first() - if thing is None: - logger.warning( - f"Thing with PointID {row.PointID} not found. Skipping water level." + f"Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to invalid date/time: {e}" ) continue @@ -87,10 +89,10 @@ def transfer_water_levels(session): # else: # collecting_organization = row.MeasuringAgency - if pd.isna(row.MeasuredBy): - sampler_name = "Unknown" - else: - sampler_name = row.MeasuredBy + # if pd.isna(row.MeasuredBy): + # sampler_name = "Unknown" + # else: + # sampler_name = row.MeasuredBy field_event = FieldEvent( thing=thing, @@ -115,6 +117,7 @@ def transfer_water_levels(session): else: sample_method = "null placeholder" + #todo: use create schema to validate data sample = Sample( field_activity=field_activity, # sampler_name=sampler_name, @@ -139,6 +142,7 @@ def transfer_water_levels(session): else: level_status = None + # TODO: use create schema to validate data observation = Observation( sensor_id=sensor_id, sample=sample, From be346075905fa5342ebf4dbb94fc10d9eee8361a Mon Sep 17 00:00:00 2001 From: jirhiker Date: Wed, 24 Sep 2025 20:14:23 +0000 Subject: [PATCH 24/47] Formatting changes --- transfers/logger.py | 5 ++++- transfers/transfer.py | 1 + transfers/waterlevels_transfer.py | 8 +++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/transfers/logger.py b/transfers/logger.py index 5c08fc2c2..2924ab646 100644 --- a/transfers/logger.py +++ b/transfers/logger.py @@ -53,9 +53,12 @@ # redirect stderr to the logger # sys.stderr = StreamToLogger(logger, logging.ERROR) + def save_log_to_bucket(): bucket = get_storage_bucket() - blob = bucket.blob(f'transfer_logs/{log_filename}') + blob = bucket.blob(f"transfer_logs/{log_filename}") blob.upload_from_filename(log_filename) logger.info(f"Uploaded log to gs://{bucket.name}/transfer_logs/{log_filename}") + + # ============= EOF ============================================= diff --git a/transfers/transfer.py b/transfers/transfer.py index 40f9a67cd..67c2bf1a9 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -142,6 +142,7 @@ def main(): # todo: move the log file to a storage bucket save_log_to_bucket() + if __name__ == "__main__": main() diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index b1f6a4c1d..6a393b7ec 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -57,8 +57,10 @@ def transfer_water_levels(session): session.commit() if pd.isna(row.DepthToWater) or pd.isna(row.DateMeasured): - logger.critical(f"Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to missing " - f"data.") + logger.critical( + f"Skipping row PointID={row.PointID}, objectid={row.OBJECTID} due to missing " + f"data." + ) continue if not pd.isna(row.TimeMeasured): @@ -117,7 +119,7 @@ def transfer_water_levels(session): else: sample_method = "null placeholder" - #todo: use create schema to validate data + # todo: use create schema to validate data sample = Sample( field_activity=field_activity, # sampler_name=sampler_name, From 00369e4cff8d1df3e1fa134b9a487dda43dec7b5 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 24 Sep 2025 10:26:21 -0600 Subject: [PATCH 25/47] WIP: first draft of thing_id query parameter for /sensor --- api/sensor.py | 9 +++++++-- tests/test_sensor.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/api/sensor.py b/api/sensor.py index 33dbe5c97..8a2569915 100644 --- a/api/sensor.py +++ b/api/sensor.py @@ -26,7 +26,7 @@ editor_dependency, viewer_dependency, ) -from db import Observation, Sample, Sensor +from db import Observation, Sample, Sensor, FieldActivity, FieldEvent, Thing from schemas.sensor import SensorResponse, CreateSensor, UpdateSensor from services.crud_helper import model_patcher, model_deleter, model_adder from services.exceptions_helper import PydanticStyleException @@ -145,9 +145,14 @@ async def get_sensors( if observed_property is not None: conditions.append(Observation.observed_property == observed_property) + # TODO: update after Deployment table is implemented if thing_id is not None: + # Observation is joined for all filters joins.append(Sample) - conditions.append(Sample.thing_id == thing_id) + joins.append(FieldActivity) + joins.append(FieldEvent) + joins.append(Thing) + conditions.append(Thing.id == thing_id) if conditions: sql = sql.join(Observation) diff --git a/tests/test_sensor.py b/tests/test_sensor.py index 0c9e5aab9..bf724693b 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -161,6 +161,33 @@ def test_get_sensors(sensor): assert data["items"][0]["notes"] == sensor.notes +# TODO: update after Deployment table is implemented +def test_get_sensors_by_thing_id( + sensor, + water_chemistry_observation, + water_chemistry_sample, + water_chemistry_field_activity, + field_event, + water_well_thing, +): + response = client.get(f"/sensor?thing_id={water_well_thing.id}") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["items"][0]["id"] == sensor.id + assert data["items"][0]["created_at"] == sensor.created_at.isoformat().replace( + "+00:00", "Z" + ) + assert data["items"][0]["release_status"] == sensor.release_status + assert data["items"][0]["name"] == sensor.name + assert data["items"][0]["model"] == sensor.model + assert data["items"][0]["serial_no"] == sensor.serial_no + assert data["items"][0]["datetime_installed"] == sensor.datetime_installed + assert data["items"][0]["datetime_removed"] == sensor.datetime_removed + assert data["items"][0]["recording_interval"] == sensor.recording_interval + assert data["items"][0]["notes"] == sensor.notes + + def test_get_sensor_by_id(sensor): response = client.get(f"/sensor/{sensor.id}") assert response.status_code == 200 From e66f5736596a15e9faebd4ca7c271082b8e61c56 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 10:04:37 -0600 Subject: [PATCH 26/47] feat: create new Deployment model --- db/deployment.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 db/deployment.py diff --git a/db/deployment.py b/db/deployment.py new file mode 100644 index 000000000..feee78832 --- /dev/null +++ b/db/deployment.py @@ -0,0 +1,47 @@ +""" +This table is an installation log that creates a many-to-many relationship +between Things and Equipment, tracking which piece of hardware was installed +at which Thing and for what period of time. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import Integer, ForeignKey, String, Date, Numeric, Text +from sqlalchemy.orm import relationship, Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.thing import Thing + from db.sensor import Sensor + + +class Deployment(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents the installation of a specific piece of equipment (Sensor) at a Thing + for a defined period. This is an Association Object. + """ + + # --- Foreign Keys --- + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.thing_id"), nullable=False + ) + sensor_id: Mapped[int] = mapped_column( + Integer, ForeignKey("sensor.sensor_id"), nullable=False + ) + + # --- Columns --- + installation_date: Mapped[Date] = mapped_column(Date, nullable=False) + removal_date: Mapped[Date] = mapped_column(Date, nullable=True) + recording_interval: Mapped[int] = mapped_column(String(50), nullable=True) + recording_interval_units: Mapped[str] = lexicon_term(nullable=True) + hanging_cable_length: Mapped[float] = mapped_column(Numeric, nullable=True) + hanging_point_height: Mapped[float] = mapped_column(Numeric, nullable=True) + hanging_point_description: Mapped[str] = mapped_column(Text, nullable=True) + notes: Mapped[str] = mapped_column(Text, nullable=True) + + # --- Relationships --- + # Many-To-One: A Deployment is for one Thing. + thing: Mapped["Thing"] = relationship("Thing", back_populates="deployments") + # Many-To-One: A Deployment is of one piece of equipment (sensor). + sensor: Mapped["Sensor"] = relationship("Sensor", back_populates="deployments") From 468fdba1c85b39714dea941c1a7e02b752b5f2f8 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 10:53:35 -0600 Subject: [PATCH 27/47] refactor: modify `mapped_column` for `recording_interval` field --- db/deployment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/deployment.py b/db/deployment.py index feee78832..94068ac57 100644 --- a/db/deployment.py +++ b/db/deployment.py @@ -1,12 +1,12 @@ """ This table is an installation log that creates a many-to-many relationship -between Things and Equipment, tracking which piece of hardware was installed +between Things and Sensors, tracking which piece of hardware was installed at which Thing and for what period of time. """ from typing import TYPE_CHECKING -from sqlalchemy import Integer, ForeignKey, String, Date, Numeric, Text +from sqlalchemy import Integer, ForeignKey, Date, Numeric, Text from sqlalchemy.orm import relationship, Mapped, mapped_column from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -33,7 +33,7 @@ class Deployment(Base, AutoBaseMixin, ReleaseMixin): # --- Columns --- installation_date: Mapped[Date] = mapped_column(Date, nullable=False) removal_date: Mapped[Date] = mapped_column(Date, nullable=True) - recording_interval: Mapped[int] = mapped_column(String(50), nullable=True) + recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) recording_interval_units: Mapped[str] = lexicon_term(nullable=True) hanging_cable_length: Mapped[float] = mapped_column(Numeric, nullable=True) hanging_point_height: Mapped[float] = mapped_column(Numeric, nullable=True) From 404968b494288a3128ab0472ef1853b99223c69a Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 12:14:01 -0600 Subject: [PATCH 28/47] feat: add `deployment` relationship and proxy to `Thing` model refactor: update `field_event` proxy to SQLAlchemy 2.0 syntax --- db/thing.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/db/thing.py b/db/thing.py index f09fa4930..7441bbf62 100644 --- a/db/thing.py +++ b/db/thing.py @@ -22,6 +22,15 @@ from db.asset import Asset from db.base import AutoBaseMixin, Base, ReleaseMixin +from typing import List, TYPE_CHECKING + +if TYPE_CHECKING: + from db.location import Location + from db.field import FieldEvent + from db.deployment import Deployment + from db.sensor import Sensor + from db.contact import Contact + class Thing(Base, AutoBaseMixin, ReleaseMixin): @@ -87,10 +96,23 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): ) ) - field_events = relationship( + # --- Relationships --- + # One-To-Many: A Thing can have many FieldEvents over time. + field_events: Mapped[List["FieldEvent"]] = relationship( "FieldEvent", back_populates="thing", cascade="all, delete-orphan", uselist=True ) + # One-To-Many: A Thing can have many Deployments of sensors (equipment) over time. + deployments: Mapped[List["Deployment"]] = relationship( + "Deployment", back_populates="thing", cascade="all, delete-orphan" + ) + + # --- Association Proxies --- + # Proxy to directly access the Sensor deployed at this Thing. + sensor: AssociationProxy[List["Sensor"]] = association_proxy( + "deployments", "sensor" + ) + class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): """ From 7ffb16eab6d2ab9d043d705c03816dd594becb2d Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 12:31:29 -0600 Subject: [PATCH 29/47] feat: add `deployment` relationship and `thing` proxy to the `Sensor` table --- db/sensor.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/db/sensor.py b/db/sensor.py index 6ab04b7c9..fe5de28b2 100644 --- a/db/sensor.py +++ b/db/sensor.py @@ -16,10 +16,18 @@ from datetime import datetime from sqlalchemy import String, Integer, DateTime +from sqlalchemy.ext.associationproxy import AssociationProxy, association_proxy from sqlalchemy.orm import relationship, mapped_column, Mapped from db.base import Base, AutoBaseMixin, ReleaseMixin +from typing import List, TYPE_CHECKING + +if TYPE_CHECKING: + from .deployment import Deployment + from .thing import Thing + from .observation import Observation + class Sensor(Base, AutoBaseMixin, ReleaseMixin): """ @@ -40,10 +48,21 @@ class Sensor(Base, AutoBaseMixin, ReleaseMixin): recording_interval: Mapped[int] = mapped_column(Integer, nullable=True) notes: Mapped[str] = mapped_column(String(50), nullable=True) - observations: Mapped[list["Observation"]] = relationship( # noqa: F821 + # --- Relationships --- + # One-To-Many: A piece of Equipment can generate many Observations. + observations: Mapped[List["Observation"]] = relationship( # noqa: F821 "Observation", back_populates="sensor", ) + # One-To-Many: A Sensor (or piece of equipment) can have many Deployments over its lifetime. + deployments: Mapped[List["Deployment"]] = relationship( + "Deployment", back_populates="sensor" + ) + + # --- Association Proxies --- + # Proxy to directly access the Things where this Equipment has been deployed. + things: AssociationProxy[List["Thing"]] = association_proxy("deployments", "thing") + # ============= EOF ============================================= From 39645fa521fddc8edb46319597c3d20bdada14e1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 12:39:12 -0600 Subject: [PATCH 30/47] refactor: import necessary classes to `Sensor` model so they can be found by relationships. --- db/sensor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/sensor.py b/db/sensor.py index fe5de28b2..bc3f243fb 100644 --- a/db/sensor.py +++ b/db/sensor.py @@ -24,9 +24,9 @@ from typing import List, TYPE_CHECKING if TYPE_CHECKING: - from .deployment import Deployment - from .thing import Thing - from .observation import Observation + from db.deployment import Deployment + from db.thing import Thing + from db.observation import Observation class Sensor(Base, AutoBaseMixin, ReleaseMixin): From 290e66f9a874a3851f782b0ff0ee7e5b503d6eac Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 13:06:45 -0600 Subject: [PATCH 31/47] refactor: update foreign key names in `Deployment` table --- db/deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/deployment.py b/db/deployment.py index 94068ac57..78eaa4ef0 100644 --- a/db/deployment.py +++ b/db/deployment.py @@ -24,10 +24,10 @@ class Deployment(Base, AutoBaseMixin, ReleaseMixin): # --- Foreign Keys --- thing_id: Mapped[int] = mapped_column( - Integer, ForeignKey("thing.thing_id"), nullable=False + Integer, ForeignKey("thing.id"), nullable=False ) sensor_id: Mapped[int] = mapped_column( - Integer, ForeignKey("sensor.sensor_id"), nullable=False + Integer, ForeignKey("sensor.id"), nullable=False ) # --- Columns --- From 691eecf73c041f327fbc4d014a1541c44dceae8b Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 13:15:24 -0600 Subject: [PATCH 32/47] feat: add `deployment` model to `__init__` file so mappers can be configured.. --- db/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/db/__init__.py b/db/__init__.py index b75475493..e448b2b9e 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -22,6 +22,7 @@ from db.asset import * from db.collabnet import * from db.contact import * +from db.deployment import * from db.geochronology import * from db.geothermal import * from db.field import * From a4e9e93b83b26ed864bb0a5ff6336ebfb405b822 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 24 Sep 2025 13:25:12 -0600 Subject: [PATCH 33/47] refactor: update proxy name in `Thing` table from `sensor` to `sensors`. --- db/thing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/thing.py b/db/thing.py index 7441bbf62..7da938744 100644 --- a/db/thing.py +++ b/db/thing.py @@ -109,7 +109,7 @@ class Thing(Base, AutoBaseMixin, ReleaseMixin): # --- Association Proxies --- # Proxy to directly access the Sensor deployed at this Thing. - sensor: AssociationProxy[List["Sensor"]] = association_proxy( + sensors: AssociationProxy[List["Sensor"]] = association_proxy( "deployments", "sensor" ) From a986d96f0b09f6baac77673d067f81565739223d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 24 Sep 2025 14:01:18 -0600 Subject: [PATCH 34/47] fix: fix thing_id query parameter for /sensor --- api/sensor.py | 15 ++++++--------- core/lexicon.json | 3 +++ tests/conftest.py | 20 ++++++++++++++++++++ tests/test_sensor.py | 6 +----- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/api/sensor.py b/api/sensor.py index 8a2569915..39dc12662 100644 --- a/api/sensor.py +++ b/api/sensor.py @@ -26,7 +26,7 @@ editor_dependency, viewer_dependency, ) -from db import Observation, Sample, Sensor, FieldActivity, FieldEvent, Thing +from db import Observation, Sensor, Deployment, Thing from schemas.sensor import SensorResponse, CreateSensor, UpdateSensor from services.crud_helper import model_patcher, model_deleter, model_adder from services.exceptions_helper import PydanticStyleException @@ -138,28 +138,25 @@ async def get_sensors( This endpoint is a placeholder and should be implemented with actual logic. """ sql = select(Sensor) - # TODO: a sensor is not yet related to observation, so this won't work at the moment if thing_id is not None or observed_property is not None: conditions = [] joins = [] if observed_property is not None: + joins.append(Observation) conditions.append(Observation.observed_property == observed_property) - # TODO: update after Deployment table is implemented if thing_id is not None: - # Observation is joined for all filters - joins.append(Sample) - joins.append(FieldActivity) - joins.append(FieldEvent) + joins.append(Deployment) joins.append(Thing) conditions.append(Thing.id == thing_id) - if conditions: - sql = sql.join(Observation) + if joins: for j in joins: sql = sql.join(j) + if conditions: sql = sql.where(and_(*conditions)) + print(sql) sql = order_sort_filter(sql, Sensor, sort=sort, order=order, filter_=filter_) return paginate(conn=session, query=sql) diff --git a/core/lexicon.json b/core/lexicon.json index 41826dcb4..14d2a1828 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -114,6 +114,9 @@ {"categories": [{"name": "unit", "description": null}], "term": "deg C", "definition": "degree Celsius"}, {"categories": [{"name": "unit", "description": null}], "term": "deg second", "definition": "degree second"}, {"categories": [{"name": "unit", "description": null}], "term": "deg minute", "definition": "degree minute"}, + {"categories": [{"name": "unit", "description": null}], "term": "second", "definition": "second"}, + {"categories": [{"name": "unit", "description": null}], "term": "minute", "definition": "minute"}, + {"categories": [{"name": "unit", "description": null}], "term": "hour", "definition": "hour"}, {"categories": [{"name": "observed_property", "description": null}], "term": "groundwater level", "definition": "groundwater level measurement" }, {"categories": [{"name": "observed_property", "description": null}], "term": "temperature", "definition": "Temperature measurement"}, diff --git a/tests/conftest.py b/tests/conftest.py index 250754c0a..fe0fe8f87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,6 +219,26 @@ def second_sensor(): session.commit() +@pytest.fixture(scope="session") +def sensor_to_water_well_thing_deployment(sensor, water_well_thing): + with session_ctx() as session: + deployment = Deployment( + sensor_id=sensor.id, + thing_id=water_well_thing.id, + installation_date="2023-01-01", + removal_date=None, + recording_interval=24, + recording_interval_units="hour", + hanging_cable_length=10, + hanging_point_height=0, + hanging_point_description="hang 10", + notes="deployment fixture", + ) + session.add(deployment) + session.commit() + yield deployment + + @pytest.fixture(scope="session") def contact(water_well_thing): with session_ctx() as session: diff --git a/tests/test_sensor.py b/tests/test_sensor.py index bf724693b..4ff87c80f 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -161,13 +161,9 @@ def test_get_sensors(sensor): assert data["items"][0]["notes"] == sensor.notes -# TODO: update after Deployment table is implemented def test_get_sensors_by_thing_id( sensor, - water_chemistry_observation, - water_chemistry_sample, - water_chemistry_field_activity, - field_event, + sensor_to_water_well_thing_deployment, water_well_thing, ): response = client.get(f"/sensor?thing_id={water_well_thing.id}") From 978c057ea724ac6cddb6a76de06ac504ed48fdfc Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 24 Sep 2025 14:07:53 -0600 Subject: [PATCH 35/47] refactor: remove print debugging statement --- api/sensor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/sensor.py b/api/sensor.py index 39dc12662..92360a4d1 100644 --- a/api/sensor.py +++ b/api/sensor.py @@ -156,7 +156,6 @@ async def get_sensors( if conditions: sql = sql.where(and_(*conditions)) - print(sql) sql = order_sort_filter(sql, Sensor, sort=sort, order=order, filter_=filter_) return paginate(conn=session, query=sql) From 351dd45e2b5930d29f5e8fc0bf55570112881a7f Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 24 Sep 2025 15:14:00 -0600 Subject: [PATCH 36/47] feat: enhance asset transfer logging and integrate asset transfer in main process --- transfers/asset_transfer.py | 1 + transfers/transfer.py | 13 +++++++------ transfers/util.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 0e2f6cd36..e0d4b158d 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -57,6 +57,7 @@ def transfer_assets(session: Session) -> None: uri, blob_name = gcs_upload(ff, bucket) add_asset(session, ff, filename, thing.id, uri, blob_name) + logger.info(f"Added asset thing.id={thing.id} thing={thing.name} uri: {uri}") def transfer_assets_testing(session: Session) -> None: diff --git a/transfers/transfer.py b/transfers/transfer.py index 67c2bf1a9..9e50e9cae 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -24,12 +24,13 @@ from core.initializers import init_lexicon from db import Base from db.engine import session_ctx + +from transfers.asset_transfer import transfer_assets from transfers.group_transfer import transfer_groups from transfers.link_ids_transfer import transfer_link_ids, transfer_link_ids_welldata from transfers.contact_transfer import transfer_contacts from transfers.sensor_transfer import init_sensor from transfers.waterlevels_transfer import transfer_water_levels - from transfers.well_transfer import transfer_wells, transfer_wellscreens from transfers.thing_transfer import ( transfer_springs, @@ -90,6 +91,9 @@ def transfer_all(sess, limit=100): timeit_direct(transfer_wells, sess, limit=limit) timeit_direct(transfer_wellscreens, sess) + message("TRANSFERRING ASSETS") + timeit_direct(transfer_assets,sess) + message("TRANSFERRING SPRINGS") timeit_direct(transfer_springs, sess, limit=limit) @@ -122,10 +126,6 @@ def transfer_all(sess, limit=100): timeit_direct(transfer_link_ids, sess) timeit_direct(transfer_link_ids_welldata, sess) - # if init or transfer_assets_flag: - # message("TRANSFERRING ASSETS") - # transfer_assets_testing(sess) - message("TRANSFERRING GROUPS") timeit_direct(transfer_groups, sess) @@ -134,13 +134,14 @@ def transfer_all(sess, limit=100): def main(): - + message('START--------------------------------------') limit = int(os.environ.get("TRANSFER_LIMIT", 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) # todo: move the log file to a storage bucket save_log_to_bucket() + message('END--------------------------------------') if __name__ == "__main__": diff --git a/transfers/util.py b/transfers/util.py index 6d9d5d544..bb8ef0f15 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -36,7 +36,7 @@ get_county_from_point, get_quad_name_from_point, ) -from logger import logger +from transfers.logger import logger def replace_nans(df: pd.DataFrame, default=None) -> pd.DataFrame: From 21ce2e81aa40d13b731b01b377b484c419d62673 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Wed, 24 Sep 2025 21:14:18 +0000 Subject: [PATCH 37/47] Formatting changes --- transfers/asset_transfer.py | 4 +++- transfers/transfer.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index e0d4b158d..33087ec49 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -57,7 +57,9 @@ def transfer_assets(session: Session) -> None: uri, blob_name = gcs_upload(ff, bucket) add_asset(session, ff, filename, thing.id, uri, blob_name) - logger.info(f"Added asset thing.id={thing.id} thing={thing.name} uri: {uri}") + logger.info( + f"Added asset thing.id={thing.id} thing={thing.name} uri: {uri}" + ) def transfer_assets_testing(session: Session) -> None: diff --git a/transfers/transfer.py b/transfers/transfer.py index 9e50e9cae..0b22383db 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -92,7 +92,7 @@ def transfer_all(sess, limit=100): timeit_direct(transfer_wellscreens, sess) message("TRANSFERRING ASSETS") - timeit_direct(transfer_assets,sess) + timeit_direct(transfer_assets, sess) message("TRANSFERRING SPRINGS") timeit_direct(transfer_springs, sess, limit=limit) @@ -134,14 +134,14 @@ def transfer_all(sess, limit=100): def main(): - message('START--------------------------------------') + message("START--------------------------------------") limit = int(os.environ.get("TRANSFER_LIMIT", 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) # todo: move the log file to a storage bucket save_log_to_bucket() - message('END--------------------------------------') + message("END--------------------------------------") if __name__ == "__main__": From 4e1fcb4a6202419c386651839e937988dbd94975 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 07:45:08 -0600 Subject: [PATCH 38/47] feat: enhance asset transfer process with well photo integration and logging improvements --- transfers/asset_transfer.py | 23 ++++++++++++++--------- transfers/well_transfer.py | 5 +++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 33087ec49..737b07b0d 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -32,7 +32,7 @@ get_storage_bucket, get_storage_client, ) -from transfers.util import get_valid_things +from transfers.util import get_valid_things, read_csv from transfers.logger import logger @@ -42,17 +42,22 @@ def transfer_assets(session: Session) -> None: bucket = get_storage_bucket(client) logger.info(f"Using bucket {bucket.name}") + well_photos = read_csv("WellPhotos") # for name in ['AR0001']: # for testing for thing in get_valid_things(session): - name = thing.name - # find images in temp bucket - logger.info(f"Processing PointID: {thing.name}") - blobs = bucket.list_blobs(prefix=f"nma-photos/{thing.name}") - # move blobs from temp to assets bucket - for srcblob in blobs: - f = srcblob.download_as_bytes() - head, filename = srcblob.name.split("/") + photos = well_photos[well_photos["PointID"] == thing.name] + if photos.empty: + photos = well_photos[well_photos["PointID"] == thing.name.replace("-", "")] + if photos.empty: + logger.info(f"No photos found for PointID: {thing.name}") + continue + for i, row in enumerate(photos.itertuples()): + photo_path = row.OLEPath + srcblob = bucket.get_blob(f'nma-photos/{photo_path}') + + head, filename = srcblob.name.split("/") + f = srcblob.download_as_bytes() ff = UploadFile(file=io.BytesIO(f), filename=filename, size=len(f)) uri, blob_name = gcs_upload(ff, bucket) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index c785d4f5b..504a4b76d 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -52,6 +52,7 @@ def transfer_wells(session, limit=0): n = len(wdf) + step = 25 start_time = time.time() for i, row in enumerate(wdf.itertuples()): pointid = row.PointID @@ -63,9 +64,9 @@ def transfer_wells(session, limit=0): logger.warning("Reached limit of %d rows. Stopping migration.", limit) break - if i and not i % 25: + if i and not i % step: logger.info( - f"Processing row {i} of {n}. {row.PointID}, avg rows per second: {i / (time.time() - start_time):.2f}" + f"Processing row {i} of {n}, avg rows per second: {step / (time.time() - start_time):.2f}" ) start_time = time.time() From f56831c485c7fcc7d5b470aab5bd06074fe61892 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Thu, 25 Sep 2025 13:45:27 +0000 Subject: [PATCH 39/47] Formatting changes --- transfers/asset_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 737b07b0d..e1b61f5f3 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -54,7 +54,7 @@ def transfer_assets(session: Session) -> None: for i, row in enumerate(photos.itertuples()): photo_path = row.OLEPath - srcblob = bucket.get_blob(f'nma-photos/{photo_path}') + srcblob = bucket.get_blob(f"nma-photos/{photo_path}") head, filename = srcblob.name.split("/") f = srcblob.download_as_bytes() From bd98268fb3a82bf2af833f7594b8b954d7fe6ba0 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 08:08:07 -0600 Subject: [PATCH 40/47] feat: add warning for missing photos during asset transfer process --- transfers/asset_transfer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index e1b61f5f3..8586c50bd 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -55,6 +55,9 @@ def transfer_assets(session: Session) -> None: for i, row in enumerate(photos.itertuples()): photo_path = row.OLEPath srcblob = bucket.get_blob(f"nma-photos/{photo_path}") + if not srcblob: + logger.crtical(f"No photo found for PointID: {thing.name}, {photo_path}") + continue head, filename = srcblob.name.split("/") f = srcblob.download_as_bytes() From c258c96593330545dabf0d6c87e1877d2e4fc799 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Thu, 25 Sep 2025 14:08:27 +0000 Subject: [PATCH 41/47] Formatting changes --- transfers/asset_transfer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 8586c50bd..62b1ee4f1 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -56,7 +56,9 @@ def transfer_assets(session: Session) -> None: photo_path = row.OLEPath srcblob = bucket.get_blob(f"nma-photos/{photo_path}") if not srcblob: - logger.crtical(f"No photo found for PointID: {thing.name}, {photo_path}") + logger.crtical( + f"No photo found for PointID: {thing.name}, {photo_path}" + ) continue head, filename = srcblob.name.split("/") From 9aa409269f9588ba1a6309a07dcbf138226994e5 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 08:24:08 -0600 Subject: [PATCH 42/47] feat: add warning for missing photos during asset transfer process --- transfers/asset_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/asset_transfer.py b/transfers/asset_transfer.py index 62b1ee4f1..749bc926f 100644 --- a/transfers/asset_transfer.py +++ b/transfers/asset_transfer.py @@ -56,7 +56,7 @@ def transfer_assets(session: Session) -> None: photo_path = row.OLEPath srcblob = bucket.get_blob(f"nma-photos/{photo_path}") if not srcblob: - logger.crtical( + logger.critical( f"No photo found for PointID: {thing.name}, {photo_path}" ) continue From 53321ef6e32562188004ab7c0c8dc79cd21edde1 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 13:58:17 -0600 Subject: [PATCH 43/47] chore: ignore transfer branch in code format workflow --- .github/workflows/format_code.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/format_code.yml b/.github/workflows/format_code.yml index 9a216503b..ceb473ce8 100644 --- a/.github/workflows/format_code.yml +++ b/.github/workflows/format_code.yml @@ -1,6 +1,8 @@ name: Code Format on: push: + branches-ignore: + - transfer permissions: contents: read jobs: From 6a446fa649ad3d27d50377c9851fe5cca91b0946 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 14:01:58 -0600 Subject: [PATCH 44/47] feat: update uv.lock --- uv.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index a801f7a46..0b7fe2a35 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]] @@ -1034,7 +1034,6 @@ requires-dist = [ { name = "multidict", specifier = "==6.6.3" }, { name = "numpy", specifier = "==2.3.2" }, { name = "packaging", specifier = "==25.0" }, - { name = "pact-python", specifier = ">=2.3.3" }, { name = "pandas", specifier = "==2.3.2" }, { name = "pandas-stubs", specifier = "==2.3.0.250703" }, { name = "pg8000", specifier = "==1.31.4" }, From 0ae9c46a1cca02d98d9c493aa329c2f80fed697b Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 14:14:50 -0600 Subject: [PATCH 45/47] fix: correct datetime formatting for TimeMeasured in waterlevels_transfer.py --- transfers/waterlevels_transfer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 6a393b7ec..6c2c1888c 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -63,13 +63,15 @@ def transfer_water_levels(session): ) continue - if not pd.isna(row.TimeMeasured): - dt_measured = f"{row.DateMeasured} {row.TimeMeasured}" - else: + if pd.isna(row.TimeMeasured): + fmt = "%Y-%m-%d %I:%M:%S %p" dt_measured = f"{row.DateMeasured} 12:00:00 AM" + else: + fmt = "%Y-%m-%d %I:%M:%S" + dt_measured = f"{row.DateMeasured} {row.TimeMeasured}" try: - dt = datetime.strptime(dt_measured, "%Y-%m-%d %I:%M:%S %p") + dt = datetime.strptime(dt_measured, fmt) dt_utc = convert_mt_to_utc(dt) except ValueError as e: logger.warning( From 979ad23d26bf55d2984ef6eeff789032240bfedb Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 14:17:16 -0600 Subject: [PATCH 46/47] fix: elevate logging level to critical for location and well creation errors --- transfers/well_transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 504a4b76d..942b8d2ea 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -73,7 +73,7 @@ def transfer_wells(session, limit=0): try: location = make_location(row) except Exception as e: - logger.warning(f"Error making location for {row.PointID}: {e}") + logger.critical(f"Error making location for {row.PointID}: {e}") continue # print(location_row) @@ -103,7 +103,7 @@ def transfer_wells(session, limit=0): ) except Exception as e: session.rollback() - logger.warning(f"Error creating well for {row.PointID}: {e}") + logger.critical(f"Error creating well for {row.PointID}: {e}") continue # TODO: use current use LUT to get well type @@ -128,7 +128,7 @@ def transfer_wells(session, limit=0): try: session.commit() except Exception as e: - logger.exception(f"Error committing well {row.PointID}: {e}") + logger.critical(f"Error committing well {row.PointID}: {e}") session.rollback() continue From 9fa5603d65e81ed71b891d635f72577ce04a0c98 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 25 Sep 2025 15:54:07 -0600 Subject: [PATCH 47/47] fix: improve datetime formatting for TimeMeasured in waterlevels_transfer.py --- transfers/waterlevels_transfer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 6c2c1888c..ac1d416f1 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -64,11 +64,16 @@ def transfer_water_levels(session): continue if pd.isna(row.TimeMeasured): - fmt = "%Y-%m-%d %I:%M:%S %p" - dt_measured = f"{row.DateMeasured} 12:00:00 AM" + fmt = "%Y-%m-%d" + dt_measured = row.DateMeasured else: - fmt = "%Y-%m-%d %I:%M:%S" - dt_measured = f"{row.DateMeasured} {row.TimeMeasured}" + fmt = "%Y-%m-%d %H:%M:%S.%f" + t = row.TimeMeasured + # Truncate microseconds to 6 digits if present + if '.' in t: + t = t[:-6] + + dt_measured = f"{row.DateMeasured} {t}" try: dt = datetime.strptime(dt_measured, fmt)