From ea5a07bacb6fca62c280f2923d18cddca2711233 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 10 Sep 2025 13:00:09 -0600 Subject: [PATCH 01/33] feat: add LU tables to transfers for mapping to lexicon --- .../data/lookup_tables/LU_AltitudeDatum.csv | 3 + .../data/lookup_tables/LU_AltitudeMethod.csv | 13 + .../lookup_tables/LU_CollectionMethod.csv | 8 + .../lookup_tables/LU_ConstructionMethod.csv | 9 + .../lookup_tables/LU_CoordinateAccuracy.csv | 11 + .../data/lookup_tables/LU_CoordinateDatum.csv | 3 + .../lookup_tables/LU_CoordinateMethod.csv | 9 + .../data/lookup_tables/LU_CurrentUse.csv | 15 + .../data/lookup_tables/LU_DataQuality.csv | 10 + .../data/lookup_tables/LU_DataSource.csv | 11 + .../LU_Depth_CompletionSource.csv | 11 + .../LU_Discharge_ChemistrySource.csv | 6 + .../data/lookup_tables/LU_FieldNoteTypes.csv | 4 + .../data/lookup_tables/LU_Formations.csv | 296 ++++++++++++++++++ .../data/lookup_tables/LU_LevelStatus.csv | 24 ++ transfers/data/lookup_tables/LU_Lithology.csv | 77 +++++ .../data/lookup_tables/LU_MajorAnalyte.csv | 23 ++ .../lookup_tables/LU_MeasurementMethod.csv | 25 ++ .../data/lookup_tables/LU_MeasuringAgency.csv | 19 ++ .../lookup_tables/LU_MinorTraceAnalyte.csv | 93 ++++++ .../lookup_tables/LU_MonitoringStatus.csv | 17 + .../data/lookup_tables/LU_SampleType.csv | 12 + transfers/data/lookup_tables/LU_SiteType.csv | 12 + transfers/data/lookup_tables/LU_Status.csv | 6 + 24 files changed, 717 insertions(+) create mode 100644 transfers/data/lookup_tables/LU_AltitudeDatum.csv create mode 100644 transfers/data/lookup_tables/LU_AltitudeMethod.csv create mode 100644 transfers/data/lookup_tables/LU_CollectionMethod.csv create mode 100644 transfers/data/lookup_tables/LU_ConstructionMethod.csv create mode 100644 transfers/data/lookup_tables/LU_CoordinateAccuracy.csv create mode 100644 transfers/data/lookup_tables/LU_CoordinateDatum.csv create mode 100644 transfers/data/lookup_tables/LU_CoordinateMethod.csv create mode 100644 transfers/data/lookup_tables/LU_CurrentUse.csv create mode 100644 transfers/data/lookup_tables/LU_DataQuality.csv create mode 100644 transfers/data/lookup_tables/LU_DataSource.csv create mode 100644 transfers/data/lookup_tables/LU_Depth_CompletionSource.csv create mode 100644 transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv create mode 100644 transfers/data/lookup_tables/LU_FieldNoteTypes.csv create mode 100644 transfers/data/lookup_tables/LU_Formations.csv create mode 100644 transfers/data/lookup_tables/LU_LevelStatus.csv create mode 100644 transfers/data/lookup_tables/LU_Lithology.csv create mode 100644 transfers/data/lookup_tables/LU_MajorAnalyte.csv create mode 100644 transfers/data/lookup_tables/LU_MeasurementMethod.csv create mode 100644 transfers/data/lookup_tables/LU_MeasuringAgency.csv create mode 100644 transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv create mode 100644 transfers/data/lookup_tables/LU_MonitoringStatus.csv create mode 100644 transfers/data/lookup_tables/LU_SampleType.csv create mode 100644 transfers/data/lookup_tables/LU_SiteType.csv create mode 100644 transfers/data/lookup_tables/LU_Status.csv diff --git a/transfers/data/lookup_tables/LU_AltitudeDatum.csv b/transfers/data/lookup_tables/LU_AltitudeDatum.csv new file mode 100644 index 000000000..7a59cb1c1 --- /dev/null +++ b/transfers/data/lookup_tables/LU_AltitudeDatum.csv @@ -0,0 +1,3 @@ +CODE,id +NAVD88,1 +NGVD29,2 diff --git a/transfers/data/lookup_tables/LU_AltitudeMethod.csv b/transfers/data/lookup_tables/LU_AltitudeMethod.csv new file mode 100644 index 000000000..3e2c56d2c --- /dev/null +++ b/transfers/data/lookup_tables/LU_AltitudeMethod.csv @@ -0,0 +1,13 @@ +CODE,MEANING,id +A,Altimeter,1 +D,Differentially corrected GPS,2 +F,Survey-grade GPS,3 +G,Global positioning system (GPS),4 +I,LiDAR DEM,5 +L,Level or other survey method,6 +M,Interpolated from topographic map,7 +N,Interpolated from digital elevation model (DEM),8 +R,Reported,9 +U,Unknown,10 +W,"Survey-grade Global Navigation Satellite Sys, Lvl1",11 +E,USGS National Elevation Dataset (NED),12 diff --git a/transfers/data/lookup_tables/LU_CollectionMethod.csv b/transfers/data/lookup_tables/LU_CollectionMethod.csv new file mode 100644 index 000000000..7a8e17699 --- /dev/null +++ b/transfers/data/lookup_tables/LU_CollectionMethod.csv @@ -0,0 +1,8 @@ +CODE,MEANING,id +B,Bailer,1 +F,Faucet at well head,2 +G,Grab sample,3 +H,Faucet or outlet at house,4 +P,Pump,5 +T,Thief sampler,6 +U,Unknown,7 diff --git a/transfers/data/lookup_tables/LU_ConstructionMethod.csv b/transfers/data/lookup_tables/LU_ConstructionMethod.csv new file mode 100644 index 000000000..3b0979c20 --- /dev/null +++ b/transfers/data/lookup_tables/LU_ConstructionMethod.csv @@ -0,0 +1,9 @@ +CODE,MEANING,id +A,Air-Rotary,1 +B,Bored or augered,2 +C,Cable-tool,3 +H,Hydraulic rotary (mud or water),4 +P,Air percussion,5 +R,Reverse rotary,6 +V,Driven,7 +Z,Other (explain in notes),8 diff --git a/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv b/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv new file mode 100644 index 000000000..4c37aba73 --- /dev/null +++ b/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv @@ -0,0 +1,11 @@ +CODE,MEANING,id +1,+/- 0.1 second (differentially corrected GPS),1 +5,+/- 0.5 second (precise handheld GPS),2 +F,+/- 5 seconds,3 +H,+/- 0.01 second (differentially corrected GPS),4 +M,+/- 1 minute,5 +R,+/- 3 seconds,6 +S,+/- 1 second,7 +T,+/- 10 seconds,8 +U,Unknown,9 +5m,5 meters,10 diff --git a/transfers/data/lookup_tables/LU_CoordinateDatum.csv b/transfers/data/lookup_tables/LU_CoordinateDatum.csv new file mode 100644 index 000000000..10cab51f3 --- /dev/null +++ b/transfers/data/lookup_tables/LU_CoordinateDatum.csv @@ -0,0 +1,3 @@ +DATUMCODE,id +NAD27,1 +NAD83,2 diff --git a/transfers/data/lookup_tables/LU_CoordinateMethod.csv b/transfers/data/lookup_tables/LU_CoordinateMethod.csv new file mode 100644 index 000000000..5eda6771f --- /dev/null +++ b/transfers/data/lookup_tables/LU_CoordinateMethod.csv @@ -0,0 +1,9 @@ +CODE,MEANING,id +D,Differentially corrected GPS,1 +F,Survey-grade global positioning system (SGPS),2 +G,"GPS, uncorrected",3 +M,Interpolated from map,4 +N,Interpolated from DEM,5 +R,Reported,6 +S,"Transit, theodolite, or other survey method",7 +U,Unknown,8 diff --git a/transfers/data/lookup_tables/LU_CurrentUse.csv b/transfers/data/lookup_tables/LU_CurrentUse.csv new file mode 100644 index 000000000..02041ff4f --- /dev/null +++ b/transfers/data/lookup_tables/LU_CurrentUse.csv @@ -0,0 +1,15 @@ +CODE,MEANING,id +A,"Open, unequipped well",1 +C,Commercial,2 +D,Domestic,3 +E,Power generation,4 +I,Irrigation,5 +L,Livestock,6 +M,Mining,7 +N,Industrial,8 +O,Observation,9 +P,Public supply,10 +S,Shared domestic,11 +T,Institutional,12 +U,Unknown,13 +X,Unused,14 diff --git a/transfers/data/lookup_tables/LU_DataQuality.csv b/transfers/data/lookup_tables/LU_DataQuality.csv new file mode 100644 index 000000000..8926c31a9 --- /dev/null +++ b/transfers/data/lookup_tables/LU_DataQuality.csv @@ -0,0 +1,10 @@ +CODE,MEANING,id,SHORT_MEANING,WebDataEntryVisible +1,Water level accurate to within two hundreths of a foot,1,Good,1 +2,Water level accurate to within one foot,2,Fair,1 +3,Water level accuracy not to nearest foot or water level not repeatable,3,Poor,1 +U0,Water level accurate to nearest foot (USGS accuracy level),4,,1 +U1,Water level accurate to nearest tenth of a foot (USGS accuracy level),5,,1 +U2,Water level accurate to nearest one-hundredth of a foot (USGS accuracy level),6,,1 +U9,Water level accuracy not to nearest foot (USGS accuracy level),7,,1 +UU,Water level accuracy unknown (USGS accuracy level),8,,1 +0,None,11,NA,1 diff --git a/transfers/data/lookup_tables/LU_DataSource.csv b/transfers/data/lookup_tables/LU_DataSource.csv new file mode 100644 index 000000000..c542870d7 --- /dev/null +++ b/transfers/data/lookup_tables/LU_DataSource.csv @@ -0,0 +1,11 @@ +CODE,MEANING,id +A,Reported by another agency,2 +D,From driller's log or well report,3 +G,"Private geologist, consultant or univ associate",4 +L,Depth interpreted fr geophys logs by source agency,5 +M,"Memory of owner, operator, driller",6 +O,Reported by owner of well,7 +R,Reported by person other than driller owner agency,8 +S,Measured by NMBGMR staff,9 +Z,Other,10 +P,Data Portal,11 diff --git a/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv b/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv new file mode 100644 index 000000000..1282d5ffe --- /dev/null +++ b/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv @@ -0,0 +1,11 @@ +CODE,MEANING,id +A,Reported by another agency,1 +D,From driller's log or well report,2 +G,"Private geologist, consultant or univ associate",3 +L,Depth interpreted fr geophys logs by source agency,4 +M,"Memory of owner, operator, driller",5 +O,Reported by owner of well,6 +R,Reported by person other than driller owner agency,7 +S,Measured by NMBGMR staff,8 +Z,Other,9 +P,Data Portal,10 diff --git a/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv b/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv new file mode 100644 index 000000000..4f5a99684 --- /dev/null +++ b/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv @@ -0,0 +1,6 @@ +CODE,MEANING,id +A,Reported by another agency,1 +G,"Private geologist, consultant, or university associate",2 +R,Information from a report,3 +S,Measured by Bureau scientist,4 +Z,Other (explain),5 diff --git a/transfers/data/lookup_tables/LU_FieldNoteTypes.csv b/transfers/data/lookup_tables/LU_FieldNoteTypes.csv new file mode 100644 index 000000000..80c5e9273 --- /dev/null +++ b/transfers/data/lookup_tables/LU_FieldNoteTypes.csv @@ -0,0 +1,4 @@ +NoteType_id,FieldNoteType +3,well inventory +4,water level +5,water chemistry diff --git a/transfers/data/lookup_tables/LU_Formations.csv b/transfers/data/lookup_tables/LU_Formations.csv new file mode 100644 index 000000000..fc8beb9f4 --- /dev/null +++ b/transfers/data/lookup_tables/LU_Formations.csv @@ -0,0 +1,296 @@ +Code,Meaning,id +000EXRV,Extrusive Rocks,1 +000IRSV,Intrusive Rocks,2 +050QUAL,Quaternary Alluvium in Valleys,3 +100QBAS,Quaternary basalt,4 +110ALVM,Quaternary Alluvium,5 +110AVMB,"Alluvium, Bolson Deposits and Other Surface Deposits",6 +110BLSN,Bolson Fill,7 +110NTGU,"Naha and Tsegi Alluvium Deposits, undifferentiated",8 +110PTODC,"Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche",10 +111MCCR,McCathys Basalt Flow,11 +112ANCH,"Upper Santa Fe Group, Ancha Formation (QTa)",12 +112CURB,Cuerbio Basalt,13 +112LAMA,"Lama Formation (QTl, QTbh) and other mountain front alluvial fans",14 +112LAMAb,"Lama Fm (QTl, QTbh) between Servilleta Basalts",15 +112LGUN,Laguna Basalt Flow,16 +112QTBF,Quaternary-Tertiary basin fill (not in valleys),17 +112QTBFlac,"Quaternary-Tertiary basin fill, lacustrian-playa lithofacies",18 +112QTBFpd,"Quaternary-Tertiary basin fill, distal piedmont lithofacies",19 +112QTBFppm,"Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies",20 +112SNTF,"Santa Fe Group, undivided",21 +112SNTFA,"Upper Santa Fe Group, axial facies",22 +112SNTFOB,"Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin",23 +112SNTFP,"Upper Santa Fe Group, piedmont facies",24 +112TRTO,Tuerto Gravels (QTt),25 +120DTIL,Datil Formation,26 +120ELRT,El Rito Formation,27 +120IRSV,Tertiary Intrusives,28 +120SBLC,"Sierra Blanca Volcanics, undivided",29 +120SRVB,Tertiary Servilletta Basalts (Tsb),30 +120SRVBf,"Tertiary Servilletta Basalts, fractured (Tsbf)",31 +120TSBV_Lower,Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm),32 +120TSBV_Upper,Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm),33 +121CHMT,Chamita Formation (Tc),34 +121CHMTv,"Chamita Fm, Vallito member (Tcv)",35 +121CHMTvs,"Chamita Fm, sandy Vallito member (Tcvs)",36 +121OGLL,Ogallala Formation,37 +121PUYEF,"Puye Conglomerate, Fanglomerate Member",38 +121TSUQ,"Tesuque Formation, undifferentiated unit",39 +121TSUQa,Tesuque Fm lithosome A (Tta),40 +121TSUQacu,"Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)",41 +121TSUQacuf,"Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)",42 +121TSUQaml,Tesuque Fm lower-middle lithosome A (Ttaml),43 +121TSUQb,Tesuque Fm lithosome B (Ttb),44 +121TSUQbfl,"Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)",45 +121TSUQbfm,"Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)",46 +121TSUQbp,"Tesuque Fm lithosome B, Pojoaque member (Ttbp)",47 +121TSUQce,"Tesuque Fm, Cejita member (Ttce)",48 +121TSUQe,Tesuque Fm lithosome E (Tte),49 +121TSUQs,Tesuque Fm lithosome S (Tts),50 +121TSUQsa,Tesuque Fm lateral gradation lithosomes S and A (Ttsag),51 +121TSUQsc,Tesuque Fm coarse-grained lithosome S (Ttsc),52 +121TSUQsf,"Tesuque Fm, fine-grained lithosome S (Ttsf)",53 +122CHOC,Chamita and Ojo Caliente interlayered (Ttoc),54 +122CRTO,"Chama El Rito Formation (Tesuque member, Ttc)",55 +122OJOC,"Ojo Caliente Formation (Tesuque member, Tto)",56 +122PICR,Picuris Tuff,57 +122PPTS,Popotosa Formation,58 +122SNTFP,"Lower Santa Fe Group, piedmont facies",59 +123DTILSPRS,"Datil Group ignimbrites and lavas and Spears Group, interbedded",60 +123DTMGandbas,"Datil and Mogollon Group andesite, basaltic andesite, and basalt flows",61 +123DTMGign,Datil and Mogollon Group ignimbrites,62 +123DTMGrhydac,Datil and Mogollon Group rhyolite and dacite flows,63 +123ESPN,T Espinaso Formation (Te),64 +123GLST,T Galisteo Formation,65 +123PICS,T Picuris Formation (Tp),66 +123PICSc,"T Picuris Formation, basal conglomerate (Tpc)",67 +123PICSl,T lower Picuris Formation (Tpl),68 +123SPRSDTMGlava,"Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded",69 +123SPRSlower,"Spears Group, lower part; tuffaceous, gravelly debris and mud flows",70 +123SPRSmid_uppe,"Spears Group, middle-upper part; excludes Dog Spring Formation",71 +124BACA,Baca Formation,72 +124CBMN,Cub Mountain Formation,73 +124LLVS,Llaves Member of San Jose Formation,74 +124PSCN,Poison Canyon Formation,75 +124RGIN,Regina Member of San Jose Formation,76 +124SNJS,San Jose Formation,77 +124TPCS,TapicitosMember of San Jose Formation,78 +125NCMN,Nacimiento Formation,79 +125NCMNS,"Nacimiento Formation, Sandy Shale Facies",80 +125RTON,Raton Formation,81 +130CALDFLOOR,Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo.,82 +180TKSCC_Upper,"Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm",83 +180TKTR,"Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T",84 +210CRCS,"Cretaceous System, undivided",85 +210GLUPC_Lower,K Gallup Sandstone and lower Crevasse Canyon Fm,86 +210HOSTD,K Hosta Dalton,87 +210MCDK,K Mancos/Dakota undivided,88 +210MNCS,"Mancos Shale, undivided",89 +210MNCSL,K Lower Mancos,90 +210MNCSU,K Upper Mancos,91 +211CLFHV,"Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co.",92 +211CRLL,Carlile Shale,93 +211CRVC,Crevasse Canyon Formation of Mesaverde Group,94 +211DKOT,Dakota Sandstone or Formation,95 +211DLCO,Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group,96 +211DLTN,Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group,97 +211FRHS,Fort Hays Limestone Member of Niobrara Formation,98 +211FRLD,Fruitland Formation,99 +211FRMG,Farmington Sandstone Member of Kirtland Shale,100 +211GBSNC,Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group,101 +211GLLG,Gallego Sandstone Member of Gallup Sandstone,102 +211GLLP,Gallup Sandstone,103 +211GRRG,Greenhorn and Graneros Formations,104 +211GRRS,Graneros Shale,105 +211HOST,Hosta Tongue of Point Lookout Sandstone of Mesaverde Group,106 +211KRLD,Kirtland Shale,107 +211LWIS,Lewis Shale,108 +211MENF,Menefee Formation,109 +211MENFU,K Upper Menefee (above Harmon Sandstone),111 +211MVRD,Mesaverde Group,112 +211OJAM,Ojo Alamo Sandstone,113 +211PCCF,Pictured Cliffs Sandstone,114 +211PIRR,Pierre Shale,115 +211PNLK,Point Lookout Sandstone,116 +211SMKH,Smoky Hill Marl Member,117 +211TLLS,Twowells Sandstone Lentil of Pike of Dakota Sandstone,118 +212KTRP,"K Dakota Sandstone, Moenkopi Fm, Artesia Group",119 +217PRGR,Purgatoire Formation,120 +220ENRD,Entrada Sandstone,121 +220JURC,Jurassic undivided,122 +220NAVJ,Navajo Sandstone,123 +221BLFF,Bluff Sandstone of Morrison Formation,124 +221CSPG,Cow Springs Sandstone of Morrison Formation,125 +221ERADU,"Entrada Sandstone of San Rafael Group, Upper",126 +221MRSN,Morrison Formation,127 +221MRSN/BBSN,Brushy Basin Member of Morrison,128 +221MRSN/JCKP,Jackpile Sandstone Member of Morrison,129 +221MRSN/RCAP,Recapture Shale Member of Morrison,130 +221MRSN/WWCN,Westwater Canyon Member of Morrison,131 +221SLWS,Salt Wash Sandstone Member of Morrison Formation,132 +221SMVL,Summerville Formation of San Rafael Group,133 +221TDLT,J Todilto,134 +221WSRC,Westwater Canyon Sandstone Member of Morrison Formation,135 +221ZUNIS,Zuni Sandstone,136 +231AGZC,Tr Agua Zarca,137 +231AGZCU,Tr Upper Agua Zarca,138 +231CHNL,Chinle Formation,139 +231CORR,Correo Sandstone Member of Chinle Formation,140 +231DCKM,Dockum Group,141 +231PFDF,Tr Petrified Forest,142 +231PFDFL,Tr Lower Petrified Forest (below middle sandstone),143 +231PFDFM,Tr Middle Petrified Forest sandstone,144 +231PFDFU,Tr Upper Petrified Forest (above middle sandstone),145 +231RCKP,Rock Point Member of Wingate Sandstone,146 +231SNRS,Santa Rosa Sandstone,147 +231SNSL,Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation,148 +231SRMP,Shinarump Member of Chinle Formation,149 +231WNGT,Wingate Sandstone,150 +260SNAN,P San Andres,151 +260SNAN_lower,Lower San Andres Formation,152 +261SNGL,P San Andres - Glorieta Sandstone in Rio Bonito member,153 +300YESO,P Yeso,154 +300YESO_lower,Lower Yeso Formation,155 +300YESO_upper,Upper Yeso Formation,156 +310ABO,P Abo,157 +310DCLL,De Chelly Sandstone Member of Cutler Formation,158 +310GLOR,Glorieta Sandstone Member of San Andres Formation (of Manzano Group),159 +310MBLC,Meseta Blanca Sandstone Member of Yeso Formation,160 +310TRRS,Torres Member of Yeso Formation,161 +310YESO,Yeso Formation,162 +310YESOG,"Yeso Formation, Manzono Group",163 +312CSTL,Castile Formation,164 +312RSLR,Rustler Formation,165 +313ARTS,Artesia Group,166 +313BLCN,Bell Canyon Formation,167 +313BRUC,Brushy Canyon Formation of Delaware Mountain Group,168 +313CKBF,Chalk Bluff Formation,169 +313CLBD,Carlsbad Limestone,170 +313CPTN,Capitan Limestone,171 +313GDLP,Guadalupian Series,172 +313GOSP,Goat Seep Dolomite,173 +313SADG,San Andres Limestone and Glorieta Sandstone,174 +313SADR,"San Andres Limestone, undivided",175 +313TNSL,Tansill Formation,176 +313YATS,"Yates Formation, Guadalupe Group",177 +315LABR,P Laborcita (Bursum),178 +315YESOABO,Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations,179 +318ABO,P Abo,180 +318BSPG,Bone Spring Limestone,181 +318JOYT,Joyita Sandstone Member of Yeso Formation,182 +318YESO,Yeso Formation,183 +319BRSM,Bursum Formation and Equivalent Rocks,184 +320HLDR,Penn Holder,185 +320PENN,Pennsylvanian undivided,186 +320SNDI,Sandia Formation,187 +321SGDC,Sangre de Cristo Formation,188 +322BEMN,Penn Beeman,189 +325GBLR,Penn Gobbler,190 +325MDER,"Madera Limestone, undivided",191 +325MDERL,Penn Lower Madera,192 +325MDERU,Penn Upper Madera,193 +325SAND,Penn Sandia,194 +326MGDL,Magdalena Group,195 +340EPRS,Espiritu Santo Formation,196 +350PZBA,Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm,197 +350PZBB,Tul Basin area - Paleozoic strata below Bursum Fm,198 +400EMBD,Embudo Granite (undifferentiated PreCambrian near Santa Fe),199 +400PCMB,Precambrian Erathem,200 +400PREC,undifferentiated PreCambrian crystalline rocks (X),201 +400PRECintr,PreCambrian crystalline rocks and local Tertiary intrusives,202 +400PRST,Priest Granite,203 +400TUSS,Tusas Granite,204 +410PRCG,PreCambrian granite (Xg),205 +410PRCGf,"PreCambrian granite, fractured (Xgf)",206 +410PRCQ,PreCambrian quartzite (Xq),207 +410PRCQf,"PreCambrian quartzite, fractured (Xqf)",208 +121GILA,Gila Conglomerate (group),209 +312DYLK,Dewey Lake Redbeds,210 +120WMVL,Wimsattville Formation,211 +313GRBG,Grayburg Formation of Artesia Group,212 +318ABOL,Abo Sandstone (Lower Tongue),213 +318ABOU,Abo Sandstone (Upper Tongue),214 +112SNTFU,"Santa Fe Group, Upper Part",215 +310FRNR,Forty-Niner Member of Rustler Formation,216 +312OCHO,Ochoan Series,217 +313AZOT,Azotea Tongue of Seven Rivers Formation,218 +313QUEN,Queen Formation,219 +319HUCO,Hueco Limestone,220 +313SVRV,Seven Rivers Formation,221 +313CABD,Carlsbad Group,222 +320GRMS,Gray Mesa Member of Madera Formation,223 +211CLRDH,Colorado Shale,224 +120BRLM,Bearwallow Mountain Andesite,225 +122RUBO,Rubio Peak Formation,226 +313SADRL,"San Andres Limestone, Lower Cherty Member",227 +313SADRU,"San Andres Limestone, Upper Clastic Member",228 +313BRNL,Bernal Formation of Artesia Group,229 +318CPDR,Chupadera Formation,230 +121BDHC,Bidahochi Formation,231 +313SADY,"San Andres Limestone and Yeso Formation, undivided",232 +221SRFLL,"San Rafael Group, Lower Part",233 +221BLUF,Bluff Sandstone of Morrison Formation,234 +221COSP,Cow Springs Sandstone of Morrison Formation,235 +317ABYS,"Abo and Yeso, undifferentiated",236 +221BRSB,Brushy Basin Shale Member of Morrison Formation,237 +310SYDR,San Ysidro Member of Yeso Formation,238 +400SDVL,Sandoval Granite,239 +221SRFL,San Rafael Group,240 +310SGRC,Sangre de Cristo Formation,241 +231TCVS,Tecovas Formation of Dockum Group,242 +211DCRS,D-Cross Tongue of Mancos Shale of Mesaverde Group,243 +211ALSN,Allison Member of Menefee Formation of Mesaverde Group,244 +211LVNN,La Ventana Tongue of Cliff House Sandstone,245 +211MORD,Madrid Formation,246 +210PRMD,Pyramid Shale,247 +124ANMS,Animas Formation,248 +211NBRR,Niobrara Formation,249 +111ALVM,Holocene Alluvium,250 +122SNTFL,"Santa Fe Group, Lower Part",251 +111CPLN,Capulin Basalts,252 +120CRSN,Carson Conflomerate,253 +111CRMS,Covered/Reclaimed Mine Spoil,254 +111CRMSA,Covered/Reclaimed Mine Spoil and Ash,255 +111SPOL,Spoil,256 +110TURT,Tuerto Gravel of Santa Fe Group,257 +221RCPR,Recapture Shale Member of Morrison Formation,258 +320BLNG,Bullington Member of Magdalena Formation,259 +112ANCHsr,"Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits",260 +121TSUQae,Tesuque Fm Lithosomes A and E,261 +230TRSC,Triassic undifferentiated,262 +122TSUQdx,"Tesuque Fm, Dixon member (Ttd)",263 +123PICSu,T upper Picuris Formation (Tpu),264 +123PICSm,T middle Picuris Formation (Tpm),265 +123PICSmc,T middle conglomerate Picuris Formation (Tpmc),266 +120VBVC,Tertiary volcanic breccia/volcaniclastic conglomerate,267 +120VCSS,Tertiary volcaniclastic sandstone,268 +124DMDT,Diamond Tail Formation,269 +325ALMT,Penn Alamitos Formation,270 +400SAND,Sandia Granite,271 +318VCPK,Victorio Peak Limestone,272 +318BSVP,Bone Spring and Victorio Peak Limestones,273 +100ALVM,Alluvium,274 +310PRMN,Permian System,275 +110AVPS,Alluvium and Permian System,276 +313CRCX,Capitan Reef Complex and Associated Limestones,277 +112SLBL,Salt Bolson,278 +112SBCRC,Salt Bolson and Capitan Reef Complex,279 +313CRDM,Capitan Reef Complex - Delaware Mountain Group,280 +112SBDM,Salt Bolson and Delaware Mountain Group,281 +120BLSN,Bolson Deposits,282 +112SBCR,Salt Bolson and Cretaceous Rocks,283 +112HCBL,Hueco Bolson,284 +120IVIG,Intrusive Rocks,285 +112RLBL,Red Light Draw Bolson,286 +112EFBL,Eagle Flat Bolson,287 +112GRBL,Green River Bolson,288 +123SAND,Sanders Canyon Formation,289 +210MRNH,Moreno Hill Formation,291 +320ALMT,Alamito Shale,292 +313DLRM,Delaware Mountain Group,293 +300PLZC,Paleozoic Erathem,294 +122SPRS,Spears Member of Datil Formation,295 +110AVTV,Alluvium and Tertiary Volcanics,296 +313DMBS,Delaware Mountain Group - Bone Spring Limestone,297 +120ERSV,Tertiary extrusives,298 diff --git a/transfers/data/lookup_tables/LU_LevelStatus.csv b/transfers/data/lookup_tables/LU_LevelStatus.csv new file mode 100644 index 000000000..435083a4a --- /dev/null +++ b/transfers/data/lookup_tables/LU_LevelStatus.csv @@ -0,0 +1,24 @@ +CODE,MEANING,id,WebDataEntryVisible +A,Water level affected by atmospheric pressure,1,0 +C,Water level was frozen (no level recorded).,2,1 +D,Site was dry,3,1 +E,Site was flowing recently.,4,1 +F,Site was flowing. Water level or head couldn't be measured w/out additional equipment.,5,1 +G,Nearby site that taps the same aquifer was flowing.,6,1 +H,Nearby site that taps the same aquifer had been flowing recently.,7,1 +I,Recharge water was being injected into the aquifer at this site.,8,0 +J,Recharge water was being injected into nearby site that taps the same aquifer.,9,0 +K,Water was cascading down the inside of the well.,10,1 +L,Water level was affected by brackish or saline water.,11,0 +M,Well was not in hydraulic contact w/formation (from source other than defined in USGS C714 or C93).,12,0 +N,Measurement was discontinued (no level recorded).,13,0 +O,Obstruction was encountered in the well (no level recorded),14,1 +P,Site was being pumped,15,1 +R,Site was pumped recently,16,1 +S,Nearby site that taps the same aquifer was being pumped,17,1 +T,Nearby site that taps the same aquifer was pumped recently,18,1 +V,Foreign substance present on the water surface,19,1 +W,Well was destroyed (no subsequent water levels should be recorded),20,1 +X,Water level affected by stage in nearby surface-water site,21,1 +Z,Other conditions exist that would affect the level (remarks),22,1 +AA,Water level not affected by status,24,0 diff --git a/transfers/data/lookup_tables/LU_Lithology.csv b/transfers/data/lookup_tables/LU_Lithology.csv new file mode 100644 index 000000000..970c3dda7 --- /dev/null +++ b/transfers/data/lookup_tables/LU_Lithology.csv @@ -0,0 +1,77 @@ +TERM,ABBREVIATION,id +Alluvium,ALVM,1 +Anhydrite,ANDR,2 +Arkose,ARKS,3 +Boulders,BLDR,4 +"Boulders, silt and clay",BLSC,5 +Boulders and sand,BLSD,6 +Bentonite,BNTN,7 +Breccia,BRCC,8 +Basalt,BSLT,9 +Conglomerate,CGLM,10 +Chalk,CHLK,11 +Chert,CHRT,12 +Clay,CLAY,13 +Caliche,CLCH,14 +Calcite,CLCT,15 +"Clay, some sand",CLSD,16 +Claystone,CLSN,17 +Coal,COAL,18 +Cobbles,COBB,19 +"Cobbles, silt and clay",COSC,20 +Cobbles and sand,COSD,21 +Dolomite,DLMT,22 +Dolomite and shale,DLSH,23 +Evaporite,EVPR,24 +Gneiss,GNSS,25 +Gypsum,GPSM,26 +Graywacke,GRCK,27 +Gravel and clay,GRCL,28 +"Gravel, cemented",GRCM,29 +"Gravel, sand and silt",GRDS,30 +"Granite, gneiss",GRGN,31 +Granite,GRNT,32 +"Gravel, silt and clay",GRSC,33 +Gravel,GRVL,34 +Igneous undifferentiated,IGNS,35 +Lignite,LGNT,36 +Limestone and dolomite,LMDM,37 +Limestone and shale,LMSH,38 +Limestone,LMSN,39 +Marl,MARL,40 +Mudstone,MDSN,41 +Metamorphic undifferentiated,MMPC,42 +Marlstone,MRLS,43 +No Recovery,NR,44 +Peat,PEAT,45 +Quartzite,QRTZ,46 +Rhyolite,RYLT,47 +Sand,SAND,48 +Schist,SCST,49 +Sand and clay,SDCL,50 +Sand and gravel,SDGL,51 +Sandstone and shale,SDSL,52 +Sand and silt,SDST,53 +"Sand, gravel and clay",SGVC,54 +Shale,SHLE,55 +Silt,SILT,56 +Siltstone and shale,SLSH,57 +Siltstone,SLSN,58 +Slate,SLTE,59 +"Sand, some clay",SNCL,60 +Sandstone,SNDS,61 +Silt and clay,STCL,62 +Travertine,TRVR,63 +Tuff,TUFF,64 +Volcanic undifferentiated,VLCC,65 +"Clay, yellow",CLYE,66 +"Clay, red",CLRD,67 +Surficial sediment,SUSE,68 +"Limestone and sandstone, interbedded",LMSD,70 +Gravel and boulders,GRBL,71 +"Sand, silt and gravel",SSTG,72 +"Sand, gravel, silt and clay",SGSC,73 +Andesite,ANDS,74 +"Ignesous, intrusive, undifferentiated",IGIN,75 +"Limestone, sandstone and shale",LMSS,76 +"Sand, silt and clay",SDCS,77 diff --git a/transfers/data/lookup_tables/LU_MajorAnalyte.csv b/transfers/data/lookup_tables/LU_MajorAnalyte.csv new file mode 100644 index 000000000..f4ae20664 --- /dev/null +++ b/transfers/data/lookup_tables/LU_MajorAnalyte.csv @@ -0,0 +1,23 @@ +CODE,MEANING,id +ALK,"Alkalinity, Total",1 +Ca,Calcium,2 +Ca(total),"Calcium, total, unfiltered",3 +Cl,Chloride,4 +CO3,Carbonate,5 +CONDLAB,"Conductivity, laboratory (µS)",6 +HCO3,Bicarbonate,7 +HRD,Hardness (CaCO3),8 +IONBAL,Ion Balance,9 +K,Potassium,10 +K(total),"Potassium, total, unfiltered",11 +Mg,Magnesium,12 +Mg(total),"Magnesium, total, unfiltered",13 +Na,Sodium,14 +Na(total),"Sodium, total, unfiltered",15 +Na+K,Sodium and Potassium combined,16 +OH,Alkalinity as OH-,17 +pHL,"pH,laboratory",18 +SO4,Sulfate,19 +TAn,Total Anions,20 +TCat,Total Cations,21 +TDS,Total Dissolved Solids,22 diff --git a/transfers/data/lookup_tables/LU_MeasurementMethod.csv b/transfers/data/lookup_tables/LU_MeasurementMethod.csv new file mode 100644 index 000000000..981ece641 --- /dev/null +++ b/transfers/data/lookup_tables/LU_MeasurementMethod.csv @@ -0,0 +1,25 @@ +CODE,MEANING,id,WebDataEntryVisible +A,Airline measurement,1,1 +B,Analog or graphic recorder,2,0 +C,Calibrated airline measurement,3,0 +D,Differential GPS; especially applicable to surface expression of ground water,4,0 +E,Estimated,5,0 +F,Transducer,6,0 +G,Pressure-gage measurement,7,1 +H,Calibrated pressure-gage measurement,8,0 +L,Interpreted from geophysical logs,9,0 +M,Manometer,10,0 +N,Non-recording gage,11,0 +O,"Observed (required for F, N, and W water level status)",12,0 +P,Sonic water level meter (acoustic pulse),13,1 +R,"Reported, method not known",14,0 +S,Steel-tape measurement,15,1 +T,Electric tape measurement (E-probe),16,1 +U,Unknown (for legacy data only; not for new data entry),17,0 +V,Calibrated electric tape; accuracy of equipment has been checked,18,0 +W,Calibrated electric cable,19,0 +X,Uncalibrated electric cable,20,0 +Z,Other (explain in notes),21,0 +K,Continuous acoustic sounder,22,0 +ZZ,Measurement not attempted,23,1 +0,null placeholder,24,0 diff --git a/transfers/data/lookup_tables/LU_MeasuringAgency.csv b/transfers/data/lookup_tables/LU_MeasuringAgency.csv new file mode 100644 index 000000000..f142a02b1 --- /dev/null +++ b/transfers/data/lookup_tables/LU_MeasuringAgency.csv @@ -0,0 +1,19 @@ +Agency,Description,id +USGS,US Geological Survey,1 +NMOSE,New Mexico Office of the State Engineer,2 +NMBGMR,New Mexico Bureau of Geology & Mineral Resources,3 +Bernalillo Cty,Bernalillo County,4 +BLM,Bureau of Land Management,5 +SFC,Santa Fe County,6 +NESWCD,Northeast Soil & Water Conservation District,7 +NMED,New Mexico Environment Dept.,8 +NMISC,New Mexico Interstate Stream Commission,9 +PVACD,Pecos Valley Artesian Conservancy District,10 +TSWCD,Taos Soil & Water Conservation District,11 +Bayard,Bayard Municipal Water,12 +OSWCD,Otero Soil & Water Conservation District,13 +SNL,Sandia National Laboratories,14 +USFS,US Forest Service,15 +TWDB,Texas Water Development Board,16 +NMT,New Mexico Tech,17 +NPS,National Park Service,18 diff --git a/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv b/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv new file mode 100644 index 000000000..6fd3653e3 --- /dev/null +++ b/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv @@ -0,0 +1,93 @@ +CODE,MEANING,id +3H,Tritium ,1 +3H:3He Age,Age of Water using dissolved gases,2 +Ag,Silver,3 +Ag(total),"Silver, total, unfiltered",4 +Al,Aluminum,5 +Al(total),"Aluminum, total, unfiltered",6 +As,Arsenic,7 +As(total),"Arsenic, total, unfiltered",8 +B,Boron,9 +B(total),"Boron, total, unfiltered",10 +Ba,Barium,11 +Ba(total),"Barium, total, unfiltered",12 +Be,Beryllium,13 +Be(total),"Beryllium, total, unfiltered",14 +Br,Bromide,15 +C13r,13C:12C ratio ,16 +C14,"14C content, pmc",17 +C14_years,Uncorrected C14 age,18 +Cd,Cadmium,19 +Cd(total),"Cadmium, total, unfiltered",20 +CFC11,Chlorofluorocarbon-11 avg age,21 +CFC113,Chlorofluorocarbon-113 avg age,22 +CFC113_12,Chlorofluorocarbon-113/12 avg RATIO age,23 +CFC12,Chlorofluorocarbon-12 avg age,24 +Co,Cobalt,25 +Co(total),"Cobalt, total, unfiltered",26 +Cr,Chromium,27 +Cr(total),"Chromium, total, unfiltered",28 +Cu,Copper,29 +Cu(total),"Copper, total, unfiltered",30 +d18O-SO4,delta O18 sulfate,31 +d34S-SO4,Sulfate 34 isotope ratio,32 +F,Fluoride,33 +Fe,Iron,34 +Fe(total),"Iron, total, unfiltered",35 +H2r,Deuterium:Hydrogen ratio ,36 +Hg,Mercury,37 +Hg(total),"Mercury, total, unfiltered",38 +Li,Lithium,39 +Li(total),"Lithium, total, unfiltered",40 +Mn,Manganese,41 +Mn(total),"Manganese, total, unfiltered",42 +Mo,Molybdenum,43 +Mo(total),"Molybdenum, total, unfiltered",44 +Ni,Nickel,45 +Ni(total),"Nickel, total, unfiltered",46 +NO2,Nitrite (as NO2),47 +NO2(N),Nitrite (as N),48 +NO3,Nitrate (as NO3),49 +NO3(N),Nitrate (as N),50 +O18r,18O:16O ratio ,51 +Pb,Lead,52 +Pb(total),"Lead, total, unfiltered",53 +PO4,Phosphate,54 +Sb,Antimony,55 +Sb(total),"Antimony, total, unfiltered",56 +Se,Selenium,57 +Se(total),"Selenium, total, unfiltered",58 +Sf6,Sulfur hexafluoride,59 +Si,Silicon,60 +Si(total),"Silicon, total, unfiltered",61 +SiO2,Silica ,62 +Sn,Tin,63 +Sn(total),"Tin, total, unfiltered",64 +Sr,Strontium,65 +Sr(total),"Strontium, total, unfiltered",66 +Sr87:Sr86,Strontium 87:86 ratio,67 +Th,Thorium,68 +Th(total),"Thorium, total, unfiltered",69 +Ti,Titanium,70 +Ti(total),"Titanium, total, unfiltered",71 +Tl,Thallium,72 +Tl(total),"Thallium, total, unfiltered",73 +U,"Uranium (total, by ICP-MS)",74 +U(total),"Uranium, total, unfiltered",75 +V,Vanadium,76 +V(total),"Vanadium, total, unfiltered",77 +Zn,Zinc,78 +Zn(total),"Zinc, total, unfiltered",79 +C14_corrected,Corrected C14 in years,80 +AsIII,Arsenite (arsenic species),81 +AsV,Arsenate (arsenic species),82 +CN6,Cyanide,83 +ERT,Estimated recharge temperature,84 +H2S,Hydrogen sulfide,85 +NH3,Ammonia,86 +NH4,Ammonium,87 +TN,Total nitrogen,88 +TKN,Total Kjeldahl nitrogen,89 +DOC,Dissolved organic carbon,90 +TOC,Total organic carbon,91 +d13C-DIC,delta C13 of dissolved inorganic carbon,93 diff --git a/transfers/data/lookup_tables/LU_MonitoringStatus.csv b/transfers/data/lookup_tables/LU_MonitoringStatus.csv new file mode 100644 index 000000000..17c1206b2 --- /dev/null +++ b/transfers/data/lookup_tables/LU_MonitoringStatus.csv @@ -0,0 +1,17 @@ +CODE,MEANING,id +6,Monitor every six months,1 +A,Annual water level,2 +B,Monitoring bi-monthly,3 +C,Monitoring complete,4 +D,Datalogger installed,5 +L,Monitor every 10 years (long-term monitor),6 +M,Monitor monthly,7 +Q,Sampling complete,8 +R,Reported to NMBGMR bimonthly,9 +S,Sample well,10 +X,Water level cannot be measured,11 +P,Repeat sampling,12 +W,Wellntel device,13 +N,Bi-annual (every other year),14 +I,Inactive,15 +H,Data share,16 diff --git a/transfers/data/lookup_tables/LU_SampleType.csv b/transfers/data/lookup_tables/LU_SampleType.csv new file mode 100644 index 000000000..698988f9d --- /dev/null +++ b/transfers/data/lookup_tables/LU_SampleType.csv @@ -0,0 +1,12 @@ +CODE,MEANING,id +BA,Background,1 +EB,Equipment blank,2 +FB,Field blank,3 +FD,Field duplicate,4 +FP,Field parameters only,5 +PR,Precipitation,6 +RE,Repeat sample,7 +SD,Standard field sample,8 +SR,Soil or Rock sample,9 +TB,Trip blank,10 +SB,Source water blank,11 diff --git a/transfers/data/lookup_tables/LU_SiteType.csv b/transfers/data/lookup_tables/LU_SiteType.csv new file mode 100644 index 000000000..e70fc2b09 --- /dev/null +++ b/transfers/data/lookup_tables/LU_SiteType.csv @@ -0,0 +1,12 @@ +CODE,MEANING,id +D,"Diversion of surface water, etc",1 +ES,Ephemeral stream,2 +GW,Groundwater other than spring (well),3 +L,"Lake, pond or reservoir",4 +M,"Meteorological (rain, snow)",5 +O,Outfall of wastewater or return flow,6 +OT,Other,7 +PS,Perennial stream,8 +R,Rock sample location,9 +S,Soil gas sample location,10 +SP,Spring,11 diff --git a/transfers/data/lookup_tables/LU_Status.csv b/transfers/data/lookup_tables/LU_Status.csv new file mode 100644 index 000000000..f596a2476 --- /dev/null +++ b/transfers/data/lookup_tables/LU_Status.csv @@ -0,0 +1,6 @@ +CODE,MEANING,id +A,Abandoned,1 +C,"Active, pumping well",2 +D,"Destroyed, exists but not usable",3 +I,"Inactive, exists but not used",4 +U,Unknown,5 From cfd88ff208aff04bcb1eea89a4db3dd3a83eba63 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 10 Sep 2025 16:47:00 -0600 Subject: [PATCH 02/33] feat: make LU to lexicon mapper for data transfer --- transfers/util.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/transfers/util.py b/transfers/util.py index 4b4cec3bb..0ef2f611f 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -220,6 +220,55 @@ def make_location(row: pd.Series) -> Location: return location +def make_lu_to_lexicon_mapper(): + lu_tables = [ + # "LU_AltitudeDatum.csv", # the code is the value, so no need for mapping + "LU_AltitudeMethod.csv", # CODE/MEANING + "LU_CollectionMethod.csv", # CODE/MEANING + "LU_ConstructionMethod.csv", # CODE/MEANING + "LU_CoordinateAccuracy.csv", # CODE/MEANING + # "LU_CoordinateDatum.csv", # the code is the value, so no need for mapping + "LU_CoordinateMethod.csv", # CODE/MEANING + "LU_CurrentUse.csv", # CODE/MEANING + "LU_DataQuality.csv", # CODE/MEANING + "LU_DataSource.csv", # CODE/MEANING + "LU_Depth_CompletionSource.csv", # CODE/MEANING + "LU_Discharge_ChemistrySource.csv", # CODE/MEANING + # "LU_FieldNoteTypes.csv", # not being used in the transfers since there are no records + # "LU_Formations.csv", # needs to be cleaned before it can be used + "LU_LevelStatus.csv", # CODE/MEANING + # "LU_Lithology.csv", # needs to be cleaned before it can be used + "LU_MajorAnalyte.csv", # CODE/MEANING + "LU_MeasurementMethod.csv", # CODE/MEANING + # "LU_MeasuringAgency.csv", # the abreviation is what is used in the new schema + "LU_MinorTraceAnalyte.csv", # CODE/MEANING + "LU_MonitoringStatus.csv", # CODE/MEANING + "LU_SampleType.csv", # CODE/MEANING + "LU_SiteType.csv", # CODE/MEANING + "LU_Status.csv", # CODE/MEANING + ] + + mappers = {} + + for lu_table in lu_tables: + p = Path("lookup_tables") / lu_table + table = read_csv(p) + + for i, row in table.iterrows(): + if lu_table == "LU_Formations.csv": + code = row.Code + meaning = row.Meaning + else: + code = row.CODE + meaning = row.MEANING + + mappers.update({code: meaning}) + return mappers + + +lu_to_lexicon_map = make_lu_to_lexicon_mapper() +print(lu_to_lexicon_map) + if __name__ == "__main__": # quad = get_quad_name_from_point(-106.5, 34.2) # print(quad) From dd970d39e81ac76a8d0eb4c863ce9e84c0a0eafb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 11 Sep 2025 15:38:45 -0600 Subject: [PATCH 03/33] refactor: move utils to services for use throughout API --- services/util.py | 98 ++++++++++++++++++++++++++++++++++++ transfers/util.py | 100 +------------------------------------ transfers/well_transfer.py | 8 +-- 3 files changed, 105 insertions(+), 101 deletions(-) create mode 100644 services/util.py diff --git a/services/util.py b/services/util.py new file mode 100644 index 000000000..658432ad7 --- /dev/null +++ b/services/util.py @@ -0,0 +1,98 @@ +from shapely.ops import transform +import pyproj +import httpx + +TRANSFORMERS = {} + + +def transform_srid(geometry, source_srid, target_srid): + """ + geometry must be a shapely geometry object, like Point, Polygon, or MultiPolygon + """ + transformer_key = (source_srid, target_srid) + if transformer_key not in TRANSFORMERS: + source_crs = pyproj.CRS(f"EPSG:{source_srid}") + target_crs = pyproj.CRS(f"EPSG:{target_srid}") + transformer = pyproj.Transformer.from_crs( + source_crs, target_crs, always_xy=True + ) + TRANSFORMERS[transformer_key] = transformer + else: + transformer = TRANSFORMERS[transformer_key] + return transform(transformer.transform, geometry) + + +def get_tiger_data( + lon: float, lat: float, layer: int, outfields: str = "*" +) -> dict | None: + url = f"https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/State_County/MapServer/{layer}/query" + params = { + "f": "json", + "where": "1=1", + "geometry": f"{lon},{lat}", + "geometryType": "esriGeometryPoint", + "inSR": "4326", + "spatialRel": "esriSpatialRelIntersects", + "outFields": outfields, + "returnGeometry": "false", + } + resp = httpx.get(url, params=params, timeout=15) + data = resp.json() + if not data.get("features"): + return None + + return data["features"][0]["attributes"] + + +def get_state_from_point(lon: float, lat: float) -> str: + attrs = get_tiger_data(lon, lat, layer=0, outfields="BASENAME") + return attrs["BASENAME"] + + +def get_county_from_point(lon: float, lat: float) -> str: + """ + Look up county for a given longitude/latitude + using the US Census TIGERWeb REST API. + """ + + attrs = get_tiger_data(lon, lat, layer=1, outfields="BASENAME") + return attrs["BASENAME"] + + +def get_quad_name_from_point(lon: float, lat: float) -> str: + url = "https://carto.nationalmap.gov/arcgis/rest/services/map_indices/MapServer/10/query" + params = { + "f": "json", + "geometry": f"{lon},{lat}", + "geometryType": "esriGeometryPoint", + "inSR": "4326", + "spatialRel": "esriSpatialRelIntersects", + "outFields": "CELL_NAME,CELL_MAPCODE", + "returnGeometry": "false", + } + + resp = httpx.get(url, params=params, timeout=15) + data = resp.json() + + if data["features"]: + attrs = data["features"][0]["attributes"] + return attrs["CELL_NAME"] + else: + print(f"No quad name found for POINT ({lon} {lat})") + return None + + +def get_epqs_elevation(lon: float, lat: float) -> float: + url = "https://epqs.nationalmap.gov/v1/json" + params = { + "x": lon, + "y": lat, + "units": "Meters", + "wkid": "4326", + "includeDate": False, + } + + resp = httpx.get(url, params=params) + data = resp.json() + + return data["value"] diff --git a/transfers/util.py b/transfers/util.py index 9f3558434..c376005b8 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -17,15 +17,14 @@ import re from pathlib import Path import logging -import httpx -import pyproj from shapely import Point -from shapely.ops import transform + from sqlalchemy.orm import Session import pandas as pd from db import Thing, Location +from services.util import transform_srid, get_epqs_elevation log_filename = f"transfers/transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" @@ -40,31 +39,12 @@ ) logger = logging.getLogger(__name__) -TRANSFORMERS = {} - def read_csv(name: str) -> pd.DataFrame: p = Path(".") / "transfers" / "data" / name return pd.read_csv(p) -def transform_srid(geometry, source_srid, target_srid): - """ - geometry must be a shapely geometry object, like Point, Polygon, or MultiPolygon - """ - transformer_key = (source_srid, target_srid) - if transformer_key not in TRANSFORMERS: - source_crs = pyproj.CRS(f"EPSG:{source_srid}") - target_crs = pyproj.CRS(f"EPSG:{target_srid}") - transformer = pyproj.Transformer.from_crs( - source_crs, target_crs, always_xy=True - ) - TRANSFORMERS[transformer_key] = transformer - else: - transformer = TRANSFORMERS[transformer_key] - return transform(transformer.transform, geometry) - - def get_valid_point_ids(session, thing_type="water well"): things = get_valid_things(session, thing_type) valid_pointids = [thing.name for thing in things] @@ -105,82 +85,6 @@ def convert_to_wgs84_vertical_datum(row, z): return z -def get_state_from_point(lon: float, lat: float) -> str: - attrs = get_tiger_data(lon, lat, layer=0, outfields="BASENAME") - return attrs["BASENAME"] - - -def get_county_from_point(lon: float, lat: float) -> str: - """ - Look up county for a given longitude/latitude - using the US Census TIGERWeb REST API. - """ - - attrs = get_tiger_data(lon, lat, layer=1, outfields="BASENAME") - return attrs["BASENAME"] - - -def get_tiger_data( - lon: float, lat: float, layer: int, outfields: str = "*" -) -> dict | None: - url = f"https://tigerweb.geo.census.gov/arcgis/rest/services/TIGERweb/State_County/MapServer/{layer}/query" - params = { - "f": "json", - "where": "1=1", - "geometry": f"{lon},{lat}", - "geometryType": "esriGeometryPoint", - "inSR": "4326", - "spatialRel": "esriSpatialRelIntersects", - "outFields": outfields, - "returnGeometry": "false", - } - resp = httpx.get(url, params=params, timeout=15) - data = resp.json() - if not data.get("features"): - return None - - return data["features"][0]["attributes"] - - -def get_quad_name_from_point(lon: float, lat: float) -> str: - url = "https://carto.nationalmap.gov/arcgis/rest/services/map_indices/MapServer/10/query" - params = { - "f": "json", - "geometry": f"{lon},{lat}", - "geometryType": "esriGeometryPoint", - "inSR": "4326", - "spatialRel": "esriSpatialRelIntersects", - "outFields": "CELL_NAME,CELL_MAPCODE", - "returnGeometry": "false", - } - - resp = httpx.get(url, params=params, timeout=15) - logger.info(resp) - data = resp.json() - - if data["features"]: - attrs = data["features"][0]["attributes"] - return attrs["CELL_NAME"] - else: - logger.warning(f"No quad name found for POINT ({lon} {lat})") - - -def get_epqs_elevation(lon: float, lat: float) -> float: - url = "https://epqs.nationalmap.gov/v1/json" - params = { - "x": lon, - "y": lat, - "units": "Meters", - "wkid": "4326", - "includeDate": False, - } - - resp = httpx.get(url, params=params) - data = resp.json() - - return data["value"] - - def make_location(row: pd.Series) -> Location: # TODO: should the altitude be fetched from USGS' diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index bcdb0ce5e..130dccc00 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -24,13 +24,15 @@ from schemas.thing import CreateWellScreen from services.lexicon_helper import add_lexicon_term from services.thing_helper import add_thing +from services.util import ( + get_state_from_point, + get_county_from_point, + get_quad_name_from_point, +) from transfers.util import ( make_location, filter_to_valid_point_ids, read_csv, - get_state_from_point, - get_county_from_point, - get_quad_name_from_point, logger, ) From dc89117eb4e9dacdab39b1fa93cad9b05b62b2a0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 11 Sep 2025 16:20:38 -0600 Subject: [PATCH 04/33] feat: set location state/county/quad from point --- api/location.py | 9 +++++++-- services/location_helper.py | 27 +++++++++++++++++++++++++++ services/util.py | 10 ++++++++++ tests/conftest.py | 6 +++--- tests/test_location.py | 13 +++++++++++-- 5 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 services/location_helper.py diff --git a/api/location.py b/api/location.py index 253663ab4..5e2b9abda 100644 --- a/api/location.py +++ b/api/location.py @@ -30,6 +30,7 @@ from services.geospatial_helper import make_within_wkt from services.query_helper import make_query, order_sort_filter, simple_get_by_id from services.crud_helper import model_patcher, model_deleter, model_adder +from services.location_helper import set_geographic_attributes from fastapi import APIRouter @@ -48,7 +49,9 @@ async def create_location( """ Create a new sample location in the database. """ - return model_adder(session, Location, location_data, user=user) + location = model_adder(session, Location, location_data, user=user) + set_geographic_attributes(session, location_data, location) + return location @router.patch( @@ -64,7 +67,9 @@ async def update_location( """ Update a sample location in the database. """ - return model_patcher(session, Location, location_id, location_data, user=user) + location = model_patcher(session, Location, location_id, location_data, user=user) + set_geographic_attributes(session, location_data, location) + return location # @router.get("/shapefile", summary="Get location as shapefile") diff --git a/services/location_helper.py b/services/location_helper.py new file mode 100644 index 000000000..92eac854b --- /dev/null +++ b/services/location_helper.py @@ -0,0 +1,27 @@ +from shapely.wkt import loads +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from db.location import Location +from services.util import ( + get_state_from_point, + get_county_from_point, + get_quad_name_from_point, +) + + +def set_geographic_attributes( + session: Session, payload: BaseModel, location: Location +) -> None: + """ + Set geographic attributes for a location based off of the point. This function + is to be used for both POST and PATCH requests. + """ + if payload.point is not None: + point = loads(payload.point) + longitude = point.x + latitude = point.y + location.state = get_state_from_point(longitude, latitude) + location.county = get_county_from_point(longitude, latitude) + location.quad_name = get_quad_name_from_point(longitude, latitude) + session.commit() diff --git a/services/util.py b/services/util.py index 658432ad7..6dff3e470 100644 --- a/services/util.py +++ b/services/util.py @@ -96,3 +96,13 @@ def get_epqs_elevation(lon: float, lat: float) -> float: data = resp.json() return data["value"] + + +if __name__ == "__main__": + x = -106.904107 + y = 34.068198 + + print(get_state_from_point(x, y)) + print(get_county_from_point(x, y)) + print(get_quad_name_from_point(x, y)) + print(get_epqs_elevation(x, y)) diff --git a/tests/conftest.py b/tests/conftest.py index 85092920e..aedc157d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,15 +11,15 @@ def location(): loc = Location( name="first location", notes="these are some test notes", - point="POINT(0 0 0)", + point="POINT(-107.949533 33.809665 2464.9)", release_status="draft", elevation_accuracy=100, elevation_method="Survey-grade GPS", coordinate_accuracy=50, coordinate_method="GPS, uncorrected", state="New Mexico", - county="Socorro", - quad_name="some NM quad", + county="Catron", + quad_name="Luera Mountains West", ) session.add(loc) session.commit() diff --git a/tests/test_location.py b/tests/test_location.py index a27ddf775..fc3811c62 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -44,7 +44,7 @@ def test_add_location(): payload = { "name": "test location", "notes": "these are some test notes", - "point": "POINT Z (10.1 10.1 0)", + "point": "POINT Z (-106.607784 35.118924 1558.8)", "release_status": "draft", "elevation_accuracy": 1.0, "elevation_method": "Survey-grade GPS", @@ -65,6 +65,9 @@ def test_add_location(): assert data["elevation_method"] == payload["elevation_method"] assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] assert data["coordinate_method"] == payload["coordinate_method"] + assert data["state"] == "New Mexico" + assert data["county"] == "Bernalillo" + assert data["quad_name"] == "Albuquerque East" # cleanup after test cleanup_post_test(Location, data["id"]) @@ -77,7 +80,7 @@ def test_update_location(location): payload = { "name": "patched name", "notes": "these are some patched notes", - "point": "POINT Z (10.1 20.2 0)", + "point": "POINT Z (-106.904107 34.068198 1408.3)", "release_status": "draft", "elevation_accuracy": 2.0, "elevation_method": "Survey-grade GPS", @@ -96,8 +99,14 @@ def test_update_location(location): assert data["elevation_method"] == payload["elevation_method"] assert data["coordinate_accuracy"] == payload["coordinate_accuracy"] assert data["coordinate_method"] == payload["coordinate_method"] + assert data["state"] == "New Mexico" + assert data["county"] == "Socorro" + assert data["quad_name"] == "Socorro" # cleanup after test + payload["state"] = location.state + payload["county"] = location.county + payload["quad_name"] = location.quad_name cleanup_patch_test(Location, payload, location) From 5dec7f8af75dd4784a818817090c7e310118926e Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 11 Sep 2025 21:30:40 -0600 Subject: [PATCH 05/33] feat: simplify deployment configurations --- .github/workflows/CD_production.yml | 5 -- .github/workflows/CD_staging.yml | 5 -- .github/workflows/dev_deploy.yml | 83 ----------------------------- 3 files changed, 93 deletions(-) delete mode 100644 .github/workflows/dev_deploy.yml diff --git a/.github/workflows/CD_production.yml b/.github/workflows/CD_production.yml index 73863baf2..bc6847138 100644 --- a/.github/workflows/CD_production.yml +++ b/.github/workflows/CD_production.yml @@ -40,11 +40,6 @@ jobs: echo "runtime: python313" >> app.yaml echo "entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" >> app.yaml echo "instance_class: F4" >> app.yaml - echo "inbound_services:" >> app.yaml - echo " - warmup" >> app.yaml - echo "automatic_scaling:" >> app.yaml - echo " min_instances: 0" >> app.yaml - echo " max_instances: 10" >> app.yaml echo "" >> app.yaml echo "env_variables:" >> app.yaml echo " MODE: \"production\"" >> app.yaml diff --git a/.github/workflows/CD_staging.yml b/.github/workflows/CD_staging.yml index 7b5d7c2b9..e2d9e83a4 100644 --- a/.github/workflows/CD_staging.yml +++ b/.github/workflows/CD_staging.yml @@ -40,11 +40,6 @@ jobs: echo "runtime: python313" >> app.yaml echo "entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" >> app.yaml echo "instance_class: F4" >> app.yaml - echo "inbound_services:" >> app.yaml - echo " - warmup" >> app.yaml - echo "automatic_scaling:" >> app.yaml - echo " min_instances: 0" >> app.yaml - echo " max_instances: 10" >> app.yaml echo "" >> app.yaml echo "env_variables:" >> app.yaml echo " MODE: \"production\"" >> app.yaml diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml deleted file mode 100644 index da47376f1..000000000 --- a/.github/workflows/dev_deploy.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: CD (Development) - -on: - push: - branches: [dev] - -permissions: - contents: write - -jobs: - staging-deploy: - - runs-on: ubuntu-latest - environment: staging - - steps: - - name: Check out source repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install uv in container - uses: astral-sh/setup-uv@v6 - with: - version: "latest" - - - name: Generate requirements.txt - run: | - uv export -o requirements.txt - - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v2' - with: - credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} - - # Uses Google Cloud Secret Manager to store secret credentials - - name: Create app.yaml - run: | - echo "service: dev-ocotillo-api" > app.yaml - echo "runtime: python313" >> app.yaml - echo "entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" >> app.yaml - echo "instance_class: F4" >> app.yaml - echo "inbound_services:" >> app.yaml - echo " - warmup" >> app.yaml - echo "automatic_scaling:" >> app.yaml - echo " min_instances: 0" >> app.yaml - echo " max_instances: 10" >> app.yaml - echo "" >> app.yaml - echo "env_variables:" >> app.yaml - echo " MODE: \"production\"" >> app.yaml - echo " DB_DRIVER: \"cloudsql\"" >> app.yaml - echo " CLOUD_SQL_INSTANCE_NAME: \"${{ secrets.CLOUD_SQL_INSTANCE_NAME }}\"" >> app.yaml - echo " CLOUD_SQL_DATABASE: \"${{ secrets.CLOUD_SQL_DATABASE }}\"" >> app.yaml - echo " CLOUD_SQL_USER: \"${{ secrets.CLOUD_SQL_USER }}\"" >> app.yaml - echo " CLOUD_SQL_PASSWORD: \"${{ secrets.CLOUD_SQL_PASSWORD }}\"" >> app.yaml - echo " GCS_SERVICE_ACCOUNT_KEY: \"${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}\"" >> app.yaml - echo " GCS_BUCKET_NAME: \"${{secrets.GCS_BUCKET_NAME}}\"" >> app.yaml - echo " AUTHENTIK_URL: \"${{secrets.AUTHENTIK_URL}}\"" >> app.yaml - echo " AUTHENTIK_CLIENT_ID: \"${{secrets.AUTHENTIK_CLIENT_ID}}\"" >> app.yaml - echo " AUTHENTIK_AUTHORIZE_URL: \"${{secrets.AUTHENTIK_AUTHORIZE_URL}}\"" >> app.yaml - echo " AUTHENTIK_TOKEN_URL: \"${{secrets.AUTHENTIK_TOKEN_URL}}\"" >> app.yaml - - - - name: Deploy to Google Cloud - run: | - gcloud app deploy app.yaml --quiet --project ${{ secrets.GCP_PROJECT_ID }} - - # Clean up old versions - delete only the oldest version, one created and one destroyed - - name: Clean up oldest version - run: | - OLDEST_VERSION=$(gcloud app versions list --service=dev-ocotillo-api --project=${{ secrets.GCP_PROJECT_ID}} --format="value(id)" --sort-by="version.createTime" | head -n 1) - if [ ! -z "$OLDEST_VERSION" ]; then - echo "Deleting oldest version: $OLDEST_VERSION" - gcloud app versions delete $OLDEST_VERSION --service=dev-ocotillo-api --project=${{ secrets.GCP_PROJECT_ID }} --quiet - echo "Deleted oldest version: $OLDEST_VERSION" - else - echo "No versions to delete" - fi - - - name: Remove app.yaml - run: | - rm app.yaml - From f3758d7773a449f15cd09995d3fccd12b305fe41 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 11 Sep 2025 22:47:10 -0600 Subject: [PATCH 06/33] feat: enhance logging and commit frequency during data transfer processes --- transfers/thing_transfer.py | 3 ++- transfers/transfer.py | 30 +++++++++++++++++++++++++++++- transfers/util.py | 12 +++++++++--- transfers/waterlevels_transfer.py | 14 ++++++++++++-- transfers/well_transfer.py | 8 +++++++- 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 306df6533..3b48bfd6b 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -33,7 +33,7 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - logger.warning(f"Reached limit of {limit} rows. Stopping migration.") break - if i and not i % 100: + if i and not i % 25: logger.info( f"Processing row {i} of {n}. {row.PointID}, avg rows per second: {i / (time.time() - start_time):.2f}" ) @@ -49,6 +49,7 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - assoc.location = location assoc.thing = spring session.add(assoc) + session.commit() def transfer_springs(session, limit=None): diff --git a/transfers/transfer.py b/transfers/transfer.py index efd71c103..9ea5d5597 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import time + from dotenv import load_dotenv load_dotenv() @@ -38,11 +40,24 @@ def erase_and_initalize(session: Session) -> None: + logger.info("Erasing existing data and initializing lexicon and sensors") + starttime = time.time() Base.metadata.drop_all(session.bind) Base.metadata.create_all(session.bind) + elapsed_time = time.time() -starttime + logger.info(f"Done erasing existing data. {elapsed_time:0.2f}s") + logger.info("Initializing lexicon and sensors") + starttime = time.time() init_lexicon() + elapsed_time = time.time() - starttime + logger.info(f"Done initializing lexicon. {elapsed_time:0.2f}s") + + starttime = time.time() init_sensor(session) + elapsed_time = time.time() - starttime + logger.info(f"Done initializing sensors. {elapsed_time:0.2f}s") + def message(msg, pad=10): @@ -69,7 +84,20 @@ def main_transfer(): cleanup_wells_flag = False - limit = 500 + transfer_well_flag = True + transfer_spring_flag = True + transfer_perennial_stream_flag = True + transfer_ephemeral_stream_flag = True + transfer_met_flag = True + transfer_contacts_flag = True + transfer_waterlevels_flag = True + transfer_link_ids_flag = True + transfer_assets_flag = True + transfer_groups_flag = True + + cleanup_wells_flag = True + + limit = 100 with session_ctx() as sess: if init: erase_and_initalize(sess) diff --git a/transfers/util.py b/transfers/util.py index 79fec1c2c..9cab9e50e 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -251,10 +251,16 @@ def make_location(row: pd.Series) -> Location: point=transformed_point.wkt, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, - elevation_method=row.AltitudeMethod, + + # TODO: map code to meaning since meaning is used as the lexicon term + # elevation_method=row.AltitudeMethod, # created_at=created_at, - coordinate_accuracy=row.CoordinateAccuracy, - coordinate_method=row.CoordinateMethod, + + # TODO: row.CoordinateAccuracy is not a float + # coordinate_accuracy=row.CoordinateAccuracy, + + # TODO: map code to meaning since meaning is used as the lexicon term + #coordinate_method=row.CoordinateMethod, nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, ) diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 5dfe41191..8e0546478 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import time import uuid from datetime import datetime @@ -28,8 +29,17 @@ def transfer_water_levels(session): wd = filter_to_valid_point_ids(session, wd) gwd = wd.groupby(["PointID"]) + start_time = time.time() for index, group in gwd: - for row in group.itertuples(): + logger.info(f"Processing PointID: {index}") + n = len(group) + for i, row in enumerate(group.itertuples()): + if i and not i % 25: + logger.info( + f"Processing row {i} of {n}. {row.PointID}, avg rows per second: {i / (time.time() - start_time):.2f}" + ) + session.commit() + if pd.isna(row.DepthToWater) or pd.isna(row.DateMeasured): logger.warning(f"Skipping row {row.Index} due to missing data.") continue @@ -67,7 +77,7 @@ def transfer_water_levels(session): obs.unit = "ft" session.add(obs) - session.commit() + session.commit() # ============= EOF ============================================= diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 19d726bdf..7c35ad740 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -43,6 +43,10 @@ def transfer_wells(session, start_index=0, limit=0): wdf = wdf[wdf["SiteType"] == "GW"] wdf = wdf[wdf["Easting"].notna() & wdf["Northing"].notna()] wdf = wdf.iloc[start_index : start_index + limit] + + wdf = wdf.replace(pd.NA, None) + wdf = wdf.replace({np.nan: None}) + n = len(wdf) start_time = time.time() results = { @@ -109,6 +113,7 @@ def transfer_wells(session, start_index=0, limit=0): made_things.append(row.PointID) results["made_things"] = made_things + session.commit() return results @@ -158,13 +163,14 @@ def transfer_wellscreens(session, limit=None): CreateWellScreen.model_validate(well_screen_data) well_screen = WellScreen(**well_screen_data) session.add(well_screen) - session.commit() except ValidationError as e: logger.warning( f"Validation error for row {i} with PointID {row.PointID}: {e}" ) continue + session.commit() + def cleanup_wells(session): locations = session.query(Location).all() From 6bdf14e37d8b8e0802e0002e743fdb222133c62d Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 12 Sep 2025 04:47:29 +0000 Subject: [PATCH 07/33] Formatting changes --- transfers/transfer.py | 3 +-- transfers/util.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 9ea5d5597..9df2c94e5 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -44,7 +44,7 @@ def erase_and_initalize(session: Session) -> None: starttime = time.time() Base.metadata.drop_all(session.bind) Base.metadata.create_all(session.bind) - elapsed_time = time.time() -starttime + elapsed_time = time.time() - starttime logger.info(f"Done erasing existing data. {elapsed_time:0.2f}s") logger.info("Initializing lexicon and sensors") @@ -59,7 +59,6 @@ def erase_and_initalize(session: Session) -> None: logger.info(f"Done initializing sensors. {elapsed_time:0.2f}s") - def message(msg, pad=10): pad = "*" * pad logger.info("") diff --git a/transfers/util.py b/transfers/util.py index 9cab9e50e..f685d6b4e 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -251,16 +251,13 @@ def make_location(row: pd.Series) -> Location: point=transformed_point.wkt, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, - # TODO: map code to meaning since meaning is used as the lexicon term # elevation_method=row.AltitudeMethod, # created_at=created_at, - # TODO: row.CoordinateAccuracy is not a float # coordinate_accuracy=row.CoordinateAccuracy, - # TODO: map code to meaning since meaning is used as the lexicon term - #coordinate_method=row.CoordinateMethod, + # coordinate_method=row.CoordinateMethod, nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, ) From f753800b01baab2f0004b724911ad175b467bbaf Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 12 Sep 2025 07:08:24 -0600 Subject: [PATCH 08/33] feat: implement replace_nans utility function and refactor data handling in transfer scripts --- transfers/contact_transfer.py | 6 +++--- transfers/link_ids_transfer.py | 6 +++++- transfers/util.py | 6 ++++++ transfers/well_transfer.py | 8 +++----- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 41a46d1eb..053f90ae3 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -15,7 +15,7 @@ # =============================================================================== import numpy as np import pandas as pd -from transfers.util import read_csv, filter_to_valid_point_ids, logger +from transfers.util import read_csv, filter_to_valid_point_ids, logger, replace_nans from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address from schemas.contact import CreateContact, CreateAddress @@ -53,8 +53,8 @@ def transfer_contacts(session): odf = odf.join(ldf.set_index("OwnerKey"), on="OwnerKey") - odf = odf.replace(pd.NA, None) - odf = odf.replace({np.nan: None}) + odf = replace_nans(odf) + odf = filter_to_valid_point_ids(session, odf) for i, row in odf.iterrows(): thing = session.query(Thing).where(Thing.name == row.PointID).first() diff --git a/transfers/link_ids_transfer.py b/transfers/link_ids_transfer.py index fc1cd8a97..18e314385 100644 --- a/transfers/link_ids_transfer.py +++ b/transfers/link_ids_transfer.py @@ -14,6 +14,8 @@ # limitations under the License. # =============================================================================== import re + +import numpy as np import pandas as pd from db import Thing, ThingIdLink @@ -21,7 +23,7 @@ filter_to_valid_point_ids, logger, extract_organization, - read_csv, + read_csv, replace_nans, ) @@ -150,6 +152,8 @@ def transfer_link_ids(session, site_type="GW"): ldf = ldf[ldf["SiteType"] == site_type] ldf = ldf[ldf["Easting"].notna() & ldf["Northing"].notna()] # ldf = ldf[ldf["AlternateSiteID"].notna()] + ldf = replace_nans(ldf) + ldf = filter_to_valid_point_ids(session, ldf) diff --git a/transfers/util.py b/transfers/util.py index f685d6b4e..a95ec1d12 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -24,6 +24,7 @@ from sqlalchemy.orm import Session import pandas as pd +import numpy as np from db import Thing, Location from services.gcs_helper import get_storage_bucket @@ -64,6 +65,11 @@ def flush(self): TRANSFORMERS = {} +def replace_nans(df: pd.DataFrame, default=None)->pd.DataFrame: + df = df.replace(pd.NA, default) + return df.replace({np.nan: default}) + + def read_csv(name: str, dtype: dict | None = None) -> pd.DataFrame: bucket = get_storage_bucket() blob = bucket.blob(f"nma_csv/{name}.csv") diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 7c35ad740..f0cca1f46 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -29,7 +29,7 @@ get_state_from_point, get_county_from_point, get_quad_name_from_point, - logger, + logger, replace_nans, ) ADDED = [] @@ -44,8 +44,7 @@ def transfer_wells(session, start_index=0, limit=0): wdf = wdf[wdf["Easting"].notna() & wdf["Northing"].notna()] wdf = wdf.iloc[start_index : start_index + limit] - wdf = wdf.replace(pd.NA, None) - wdf = wdf.replace({np.nan: None}) + wdf = replace_nans(wdf) n = len(wdf) start_time = time.time() @@ -119,8 +118,7 @@ def transfer_wells(session, start_index=0, limit=0): def transfer_wellscreens(session, limit=None): wdf = read_csv("WellScreens") - wdf = wdf.replace(pd.NA, None) - wdf = wdf.replace({np.nan: None}) + wdf = replace_nans(wdf) wdf = filter_to_valid_point_ids(session, wdf) From 54772a3d7fb88544def4b358ec5c7e96606b364b Mon Sep 17 00:00:00 2001 From: jirhiker Date: Fri, 12 Sep 2025 13:08:45 +0000 Subject: [PATCH 09/33] Formatting changes --- transfers/link_ids_transfer.py | 4 ++-- transfers/util.py | 2 +- transfers/well_transfer.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/transfers/link_ids_transfer.py b/transfers/link_ids_transfer.py index 18e314385..987fac17d 100644 --- a/transfers/link_ids_transfer.py +++ b/transfers/link_ids_transfer.py @@ -23,7 +23,8 @@ filter_to_valid_point_ids, logger, extract_organization, - read_csv, replace_nans, + read_csv, + replace_nans, ) @@ -154,7 +155,6 @@ def transfer_link_ids(session, site_type="GW"): # ldf = ldf[ldf["AlternateSiteID"].notna()] ldf = replace_nans(ldf) - ldf = filter_to_valid_point_ids(session, ldf) for i, row in enumerate(ldf.itertuples()): diff --git a/transfers/util.py b/transfers/util.py index a95ec1d12..7b56bcacc 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -65,7 +65,7 @@ def flush(self): TRANSFORMERS = {} -def replace_nans(df: pd.DataFrame, default=None)->pd.DataFrame: +def replace_nans(df: pd.DataFrame, default=None) -> pd.DataFrame: df = df.replace(pd.NA, default) return df.replace({np.nan: default}) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index f0cca1f46..21b0e977c 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -29,7 +29,8 @@ get_state_from_point, get_county_from_point, get_quad_name_from_point, - logger, replace_nans, + logger, + replace_nans, ) ADDED = [] From 6c0e314b3af97e8c292b57f8a99cf35b011181da Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 08:39:36 -0600 Subject: [PATCH 10/33] feat: set elevation from national map if none in nma --- services/util.py | 4 ++-- transfers/util.py | 48 +++++++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/services/util.py b/services/util.py index 6dff3e470..274507b95 100644 --- a/services/util.py +++ b/services/util.py @@ -82,7 +82,7 @@ def get_quad_name_from_point(lon: float, lat: float) -> str: return None -def get_epqs_elevation(lon: float, lat: float) -> float: +def get_epqs_elevation_from_point(lon: float, lat: float) -> float: url = "https://epqs.nationalmap.gov/v1/json" params = { "x": lon, @@ -105,4 +105,4 @@ def get_epqs_elevation(lon: float, lat: float) -> float: print(get_state_from_point(x, y)) print(get_county_from_point(x, y)) print(get_quad_name_from_point(x, y)) - print(get_epqs_elevation(x, y)) + print(get_epqs_elevation_from_point(x, y)) diff --git a/transfers/util.py b/transfers/util.py index c376005b8..c6090f262 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -24,7 +24,13 @@ import pandas as pd from db import Thing, Location -from services.util import transform_srid, get_epqs_elevation +from services.util import ( + transform_srid, + get_epqs_elevation_from_point, + get_state_from_point, + get_county_from_point, + get_quad_name_from_point, +) log_filename = f"transfers/transfer_{datetime.now():%Y-%m-%dT%Hh%Mm%Ss}.log" @@ -89,33 +95,28 @@ def make_location(row: pd.Series) -> Location: # TODO: should the altitude be fetched from USGS' # Elevation Point Query Service https://epqs.nationalmap.gov/v1/docs - xypoint = transform_srid( - Point(row.Easting, row.Northing), - source_srid=26913, - target_srid=4326, # WGS84 SRID - ) - - z = 0 - - # idx = row.index - # idx = df.index.get_loc(row.name) - # print('asdfa', idx, row.name) - # if not z: - # z = get_epqs_elevation(xypoint.x, xypoint.y) - # z = row.Altitude if row.Altitude else 0 - # convert z from ft to meters - z = z * 0.3048 - - point = Point(row.Easting, row.Northing, z) + point = Point(row.Easting, row.Northing) # Convert the point to a WGS84 coordinate system transformed_point = transform_srid( point, source_srid=26913, target_srid=4326 # WGS84 SRID ) - # TODO: Add tests for these functions. move to a different location - # use in Location API + state = get_state_from_point(transformed_point.x, transformed_point.y) + county = get_county_from_point(transformed_point.x, transformed_point.y) + quad_name = get_quad_name_from_point(transformed_point.x, transformed_point.y) + + z = row.Altitude + if z: + z = z * 0.3048 + else: + logger.info( + f"Location {row.PointID} has no Altitude. Setting from National Map EPQS for " + ) + z = get_epqs_elevation_from_point(transformed_point.x, transformed_point.y) + + setattr(point, z, z) # TODO: determine correct created_at value # created_at = row.DateCreated @@ -133,6 +134,9 @@ def make_location(row: pd.Series) -> Location: coordinate_method=row.CoordinateMethod, nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, + state=state, + county=county, + quad_name=quad_name, ) return location @@ -144,7 +148,7 @@ def make_location(row: pd.Series) -> Location: # print(state) # county = get_county_from_point(-106.5, 34.2) # print(county) - z = get_epqs_elevation(-106.5, 34.2) + z = get_epqs_elevation_from_point(-106.5, 34.2) print(z) # ============= EOF ============================================= From 948b17bba5dbd6096b02e52fefe918f0e3dd8f00 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 08:45:14 -0600 Subject: [PATCH 11/33] feat: map lu tables to lexicon for transfer --- transfers/util.py | 54 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index cc2f37519..59c420b92 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -272,40 +272,39 @@ def make_location(row: pd.Series) -> Location: def make_lu_to_lexicon_mapper(): lu_tables = [ - # "LU_AltitudeDatum.csv", # the code is the value, so no need for mapping - "LU_AltitudeMethod.csv", # CODE/MEANING - "LU_CollectionMethod.csv", # CODE/MEANING - "LU_ConstructionMethod.csv", # CODE/MEANING - "LU_CoordinateAccuracy.csv", # CODE/MEANING - # "LU_CoordinateDatum.csv", # the code is the value, so no need for mapping - "LU_CoordinateMethod.csv", # CODE/MEANING - "LU_CurrentUse.csv", # CODE/MEANING - "LU_DataQuality.csv", # CODE/MEANING - "LU_DataSource.csv", # CODE/MEANING - "LU_Depth_CompletionSource.csv", # CODE/MEANING - "LU_Discharge_ChemistrySource.csv", # CODE/MEANING - # "LU_FieldNoteTypes.csv", # not being used in the transfers since there are no records - # "LU_Formations.csv", # needs to be cleaned before it can be used - "LU_LevelStatus.csv", # CODE/MEANING - # "LU_Lithology.csv", # needs to be cleaned before it can be used - "LU_MajorAnalyte.csv", # CODE/MEANING - "LU_MeasurementMethod.csv", # CODE/MEANING - # "LU_MeasuringAgency.csv", # the abreviation is what is used in the new schema - "LU_MinorTraceAnalyte.csv", # CODE/MEANING - "LU_MonitoringStatus.csv", # CODE/MEANING - "LU_SampleType.csv", # CODE/MEANING - "LU_SiteType.csv", # CODE/MEANING - "LU_Status.csv", # CODE/MEANING + # "LU_AltitudeDatum", # the code is the value, so no need for mapping + "LU_AltitudeMethod", # CODE/MEANING + "LU_CollectionMethod", # CODE/MEANING + "LU_ConstructionMethod", # CODE/MEANING + "LU_CoordinateAccuracy", # CODE/MEANING + # "LU_CoordinateDatum", # the code is the value, so no need for mapping + "LU_CoordinateMethod", # CODE/MEANING + "LU_CurrentUse", # CODE/MEANING + "LU_DataQuality", # CODE/MEANING + "LU_DataSource", # CODE/MEANING + "LU_Depth_CompletionSource", # CODE/MEANING + "LU_Discharge_ChemistrySource", # CODE/MEANING + # "LU_FieldNoteTypes", # not being used in the transfers since there are no records + # "LU_Formations", # needs to be cleaned before it can be used + "LU_LevelStatus", # CODE/MEANING + # "LU_Lithology", # needs to be cleaned before it can be used + "LU_MajorAnalyte", # CODE/MEANING + "LU_MeasurementMethod", # CODE/MEANING + # "LU_MeasuringAgency", # the abreviation is what is used in the new schema + "LU_MinorTraceAnalyte", # CODE/MEANING + "LU_MonitoringStatus", # CODE/MEANING + "LU_SampleType", # CODE/MEANING + "LU_SiteType", # CODE/MEANING + "LU_Status", # CODE/MEANING ] mappers = {} for lu_table in lu_tables: - p = Path("lookup_tables") / lu_table - table = read_csv(p) + table = read_csv(lu_table) for i, row in table.iterrows(): - if lu_table == "LU_Formations.csv": + if lu_table == "LU_Formations": code = row.Code meaning = row.Meaning else: @@ -317,7 +316,6 @@ def make_lu_to_lexicon_mapper(): lu_to_lexicon_map = make_lu_to_lexicon_mapper() -print(lu_to_lexicon_map) if __name__ == "__main__": # quad = get_quad_name_from_point(-106.5, 34.2) From db3a85784d7131fca10cfa10cebd8e0e1208472e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 08:47:31 -0600 Subject: [PATCH 12/33] fix: delete LU tables locally - get from cloud --- .../data/lookup_tables/LU_AltitudeDatum.csv | 3 - .../data/lookup_tables/LU_AltitudeMethod.csv | 13 - .../lookup_tables/LU_CollectionMethod.csv | 8 - .../lookup_tables/LU_ConstructionMethod.csv | 9 - .../lookup_tables/LU_CoordinateAccuracy.csv | 11 - .../data/lookup_tables/LU_CoordinateDatum.csv | 3 - .../lookup_tables/LU_CoordinateMethod.csv | 9 - .../data/lookup_tables/LU_CurrentUse.csv | 15 - .../data/lookup_tables/LU_DataQuality.csv | 10 - .../data/lookup_tables/LU_DataSource.csv | 11 - .../LU_Depth_CompletionSource.csv | 11 - .../LU_Discharge_ChemistrySource.csv | 6 - .../data/lookup_tables/LU_FieldNoteTypes.csv | 4 - .../data/lookup_tables/LU_Formations.csv | 296 ------------------ .../data/lookup_tables/LU_LevelStatus.csv | 24 -- transfers/data/lookup_tables/LU_Lithology.csv | 77 ----- .../data/lookup_tables/LU_MajorAnalyte.csv | 23 -- .../lookup_tables/LU_MeasurementMethod.csv | 25 -- .../data/lookup_tables/LU_MeasuringAgency.csv | 19 -- .../lookup_tables/LU_MinorTraceAnalyte.csv | 93 ------ .../lookup_tables/LU_MonitoringStatus.csv | 17 - .../data/lookup_tables/LU_SampleType.csv | 12 - transfers/data/lookup_tables/LU_SiteType.csv | 12 - transfers/data/lookup_tables/LU_Status.csv | 6 - 24 files changed, 717 deletions(-) delete mode 100644 transfers/data/lookup_tables/LU_AltitudeDatum.csv delete mode 100644 transfers/data/lookup_tables/LU_AltitudeMethod.csv delete mode 100644 transfers/data/lookup_tables/LU_CollectionMethod.csv delete mode 100644 transfers/data/lookup_tables/LU_ConstructionMethod.csv delete mode 100644 transfers/data/lookup_tables/LU_CoordinateAccuracy.csv delete mode 100644 transfers/data/lookup_tables/LU_CoordinateDatum.csv delete mode 100644 transfers/data/lookup_tables/LU_CoordinateMethod.csv delete mode 100644 transfers/data/lookup_tables/LU_CurrentUse.csv delete mode 100644 transfers/data/lookup_tables/LU_DataQuality.csv delete mode 100644 transfers/data/lookup_tables/LU_DataSource.csv delete mode 100644 transfers/data/lookup_tables/LU_Depth_CompletionSource.csv delete mode 100644 transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv delete mode 100644 transfers/data/lookup_tables/LU_FieldNoteTypes.csv delete mode 100644 transfers/data/lookup_tables/LU_Formations.csv delete mode 100644 transfers/data/lookup_tables/LU_LevelStatus.csv delete mode 100644 transfers/data/lookup_tables/LU_Lithology.csv delete mode 100644 transfers/data/lookup_tables/LU_MajorAnalyte.csv delete mode 100644 transfers/data/lookup_tables/LU_MeasurementMethod.csv delete mode 100644 transfers/data/lookup_tables/LU_MeasuringAgency.csv delete mode 100644 transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv delete mode 100644 transfers/data/lookup_tables/LU_MonitoringStatus.csv delete mode 100644 transfers/data/lookup_tables/LU_SampleType.csv delete mode 100644 transfers/data/lookup_tables/LU_SiteType.csv delete mode 100644 transfers/data/lookup_tables/LU_Status.csv diff --git a/transfers/data/lookup_tables/LU_AltitudeDatum.csv b/transfers/data/lookup_tables/LU_AltitudeDatum.csv deleted file mode 100644 index 7a59cb1c1..000000000 --- a/transfers/data/lookup_tables/LU_AltitudeDatum.csv +++ /dev/null @@ -1,3 +0,0 @@ -CODE,id -NAVD88,1 -NGVD29,2 diff --git a/transfers/data/lookup_tables/LU_AltitudeMethod.csv b/transfers/data/lookup_tables/LU_AltitudeMethod.csv deleted file mode 100644 index 3e2c56d2c..000000000 --- a/transfers/data/lookup_tables/LU_AltitudeMethod.csv +++ /dev/null @@ -1,13 +0,0 @@ -CODE,MEANING,id -A,Altimeter,1 -D,Differentially corrected GPS,2 -F,Survey-grade GPS,3 -G,Global positioning system (GPS),4 -I,LiDAR DEM,5 -L,Level or other survey method,6 -M,Interpolated from topographic map,7 -N,Interpolated from digital elevation model (DEM),8 -R,Reported,9 -U,Unknown,10 -W,"Survey-grade Global Navigation Satellite Sys, Lvl1",11 -E,USGS National Elevation Dataset (NED),12 diff --git a/transfers/data/lookup_tables/LU_CollectionMethod.csv b/transfers/data/lookup_tables/LU_CollectionMethod.csv deleted file mode 100644 index 7a8e17699..000000000 --- a/transfers/data/lookup_tables/LU_CollectionMethod.csv +++ /dev/null @@ -1,8 +0,0 @@ -CODE,MEANING,id -B,Bailer,1 -F,Faucet at well head,2 -G,Grab sample,3 -H,Faucet or outlet at house,4 -P,Pump,5 -T,Thief sampler,6 -U,Unknown,7 diff --git a/transfers/data/lookup_tables/LU_ConstructionMethod.csv b/transfers/data/lookup_tables/LU_ConstructionMethod.csv deleted file mode 100644 index 3b0979c20..000000000 --- a/transfers/data/lookup_tables/LU_ConstructionMethod.csv +++ /dev/null @@ -1,9 +0,0 @@ -CODE,MEANING,id -A,Air-Rotary,1 -B,Bored or augered,2 -C,Cable-tool,3 -H,Hydraulic rotary (mud or water),4 -P,Air percussion,5 -R,Reverse rotary,6 -V,Driven,7 -Z,Other (explain in notes),8 diff --git a/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv b/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv deleted file mode 100644 index 4c37aba73..000000000 --- a/transfers/data/lookup_tables/LU_CoordinateAccuracy.csv +++ /dev/null @@ -1,11 +0,0 @@ -CODE,MEANING,id -1,+/- 0.1 second (differentially corrected GPS),1 -5,+/- 0.5 second (precise handheld GPS),2 -F,+/- 5 seconds,3 -H,+/- 0.01 second (differentially corrected GPS),4 -M,+/- 1 minute,5 -R,+/- 3 seconds,6 -S,+/- 1 second,7 -T,+/- 10 seconds,8 -U,Unknown,9 -5m,5 meters,10 diff --git a/transfers/data/lookup_tables/LU_CoordinateDatum.csv b/transfers/data/lookup_tables/LU_CoordinateDatum.csv deleted file mode 100644 index 10cab51f3..000000000 --- a/transfers/data/lookup_tables/LU_CoordinateDatum.csv +++ /dev/null @@ -1,3 +0,0 @@ -DATUMCODE,id -NAD27,1 -NAD83,2 diff --git a/transfers/data/lookup_tables/LU_CoordinateMethod.csv b/transfers/data/lookup_tables/LU_CoordinateMethod.csv deleted file mode 100644 index 5eda6771f..000000000 --- a/transfers/data/lookup_tables/LU_CoordinateMethod.csv +++ /dev/null @@ -1,9 +0,0 @@ -CODE,MEANING,id -D,Differentially corrected GPS,1 -F,Survey-grade global positioning system (SGPS),2 -G,"GPS, uncorrected",3 -M,Interpolated from map,4 -N,Interpolated from DEM,5 -R,Reported,6 -S,"Transit, theodolite, or other survey method",7 -U,Unknown,8 diff --git a/transfers/data/lookup_tables/LU_CurrentUse.csv b/transfers/data/lookup_tables/LU_CurrentUse.csv deleted file mode 100644 index 02041ff4f..000000000 --- a/transfers/data/lookup_tables/LU_CurrentUse.csv +++ /dev/null @@ -1,15 +0,0 @@ -CODE,MEANING,id -A,"Open, unequipped well",1 -C,Commercial,2 -D,Domestic,3 -E,Power generation,4 -I,Irrigation,5 -L,Livestock,6 -M,Mining,7 -N,Industrial,8 -O,Observation,9 -P,Public supply,10 -S,Shared domestic,11 -T,Institutional,12 -U,Unknown,13 -X,Unused,14 diff --git a/transfers/data/lookup_tables/LU_DataQuality.csv b/transfers/data/lookup_tables/LU_DataQuality.csv deleted file mode 100644 index 8926c31a9..000000000 --- a/transfers/data/lookup_tables/LU_DataQuality.csv +++ /dev/null @@ -1,10 +0,0 @@ -CODE,MEANING,id,SHORT_MEANING,WebDataEntryVisible -1,Water level accurate to within two hundreths of a foot,1,Good,1 -2,Water level accurate to within one foot,2,Fair,1 -3,Water level accuracy not to nearest foot or water level not repeatable,3,Poor,1 -U0,Water level accurate to nearest foot (USGS accuracy level),4,,1 -U1,Water level accurate to nearest tenth of a foot (USGS accuracy level),5,,1 -U2,Water level accurate to nearest one-hundredth of a foot (USGS accuracy level),6,,1 -U9,Water level accuracy not to nearest foot (USGS accuracy level),7,,1 -UU,Water level accuracy unknown (USGS accuracy level),8,,1 -0,None,11,NA,1 diff --git a/transfers/data/lookup_tables/LU_DataSource.csv b/transfers/data/lookup_tables/LU_DataSource.csv deleted file mode 100644 index c542870d7..000000000 --- a/transfers/data/lookup_tables/LU_DataSource.csv +++ /dev/null @@ -1,11 +0,0 @@ -CODE,MEANING,id -A,Reported by another agency,2 -D,From driller's log or well report,3 -G,"Private geologist, consultant or univ associate",4 -L,Depth interpreted fr geophys logs by source agency,5 -M,"Memory of owner, operator, driller",6 -O,Reported by owner of well,7 -R,Reported by person other than driller owner agency,8 -S,Measured by NMBGMR staff,9 -Z,Other,10 -P,Data Portal,11 diff --git a/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv b/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv deleted file mode 100644 index 1282d5ffe..000000000 --- a/transfers/data/lookup_tables/LU_Depth_CompletionSource.csv +++ /dev/null @@ -1,11 +0,0 @@ -CODE,MEANING,id -A,Reported by another agency,1 -D,From driller's log or well report,2 -G,"Private geologist, consultant or univ associate",3 -L,Depth interpreted fr geophys logs by source agency,4 -M,"Memory of owner, operator, driller",5 -O,Reported by owner of well,6 -R,Reported by person other than driller owner agency,7 -S,Measured by NMBGMR staff,8 -Z,Other,9 -P,Data Portal,10 diff --git a/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv b/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv deleted file mode 100644 index 4f5a99684..000000000 --- a/transfers/data/lookup_tables/LU_Discharge_ChemistrySource.csv +++ /dev/null @@ -1,6 +0,0 @@ -CODE,MEANING,id -A,Reported by another agency,1 -G,"Private geologist, consultant, or university associate",2 -R,Information from a report,3 -S,Measured by Bureau scientist,4 -Z,Other (explain),5 diff --git a/transfers/data/lookup_tables/LU_FieldNoteTypes.csv b/transfers/data/lookup_tables/LU_FieldNoteTypes.csv deleted file mode 100644 index 80c5e9273..000000000 --- a/transfers/data/lookup_tables/LU_FieldNoteTypes.csv +++ /dev/null @@ -1,4 +0,0 @@ -NoteType_id,FieldNoteType -3,well inventory -4,water level -5,water chemistry diff --git a/transfers/data/lookup_tables/LU_Formations.csv b/transfers/data/lookup_tables/LU_Formations.csv deleted file mode 100644 index fc8beb9f4..000000000 --- a/transfers/data/lookup_tables/LU_Formations.csv +++ /dev/null @@ -1,296 +0,0 @@ -Code,Meaning,id -000EXRV,Extrusive Rocks,1 -000IRSV,Intrusive Rocks,2 -050QUAL,Quaternary Alluvium in Valleys,3 -100QBAS,Quaternary basalt,4 -110ALVM,Quaternary Alluvium,5 -110AVMB,"Alluvium, Bolson Deposits and Other Surface Deposits",6 -110BLSN,Bolson Fill,7 -110NTGU,"Naha and Tsegi Alluvium Deposits, undifferentiated",8 -110PTODC,"Pediment, Terrace and Other Deposits of Gravel, Sand and Caliche",10 -111MCCR,McCathys Basalt Flow,11 -112ANCH,"Upper Santa Fe Group, Ancha Formation (QTa)",12 -112CURB,Cuerbio Basalt,13 -112LAMA,"Lama Formation (QTl, QTbh) and other mountain front alluvial fans",14 -112LAMAb,"Lama Fm (QTl, QTbh) between Servilleta Basalts",15 -112LGUN,Laguna Basalt Flow,16 -112QTBF,Quaternary-Tertiary basin fill (not in valleys),17 -112QTBFlac,"Quaternary-Tertiary basin fill, lacustrian-playa lithofacies",18 -112QTBFpd,"Quaternary-Tertiary basin fill, distal piedmont lithofacies",19 -112QTBFppm,"Quaternary-Tertiary basin fill, proximal and medial piedmont lithofacies",20 -112SNTF,"Santa Fe Group, undivided",21 -112SNTFA,"Upper Santa Fe Group, axial facies",22 -112SNTFOB,"Upper SantaFe Group, Loma Barbon member of Arroyo Ojito Formatin",23 -112SNTFP,"Upper Santa Fe Group, piedmont facies",24 -112TRTO,Tuerto Gravels (QTt),25 -120DTIL,Datil Formation,26 -120ELRT,El Rito Formation,27 -120IRSV,Tertiary Intrusives,28 -120SBLC,"Sierra Blanca Volcanics, undivided",29 -120SRVB,Tertiary Servilletta Basalts (Tsb),30 -120SRVBf,"Tertiary Servilletta Basalts, fractured (Tsbf)",31 -120TSBV_Lower,Tertiary Sierra Blanca area lower volcanic unit (Hog Pen Fm),32 -120TSBV_Upper,Tertiary Sierra Blanca area upper volcanic unit (above Hog Pen Fm),33 -121CHMT,Chamita Formation (Tc),34 -121CHMTv,"Chamita Fm, Vallito member (Tcv)",35 -121CHMTvs,"Chamita Fm, sandy Vallito member (Tcvs)",36 -121OGLL,Ogallala Formation,37 -121PUYEF,"Puye Conglomerate, Fanglomerate Member",38 -121TSUQ,"Tesuque Formation, undifferentiated unit",39 -121TSUQa,Tesuque Fm lithosome A (Tta),40 -121TSUQacu,"Tesuque Fm (upper), Cuarteles member lithosome A (Ttacu)",41 -121TSUQacuf,"Tesuque Fm (upper), fine-grained Cuarteles member lithosome A (Ttacuf)",42 -121TSUQaml,Tesuque Fm lower-middle lithosome A (Ttaml),43 -121TSUQb,Tesuque Fm lithosome B (Ttb),44 -121TSUQbfl,"Tesuque Fm lower lithosome B, basin-floor deposits (Ttbfl)",45 -121TSUQbfm,"Tesuque Fm middle lithosome B, basin-floor deposits (Ttbfm)",46 -121TSUQbp,"Tesuque Fm lithosome B, Pojoaque member (Ttbp)",47 -121TSUQce,"Tesuque Fm, Cejita member (Ttce)",48 -121TSUQe,Tesuque Fm lithosome E (Tte),49 -121TSUQs,Tesuque Fm lithosome S (Tts),50 -121TSUQsa,Tesuque Fm lateral gradation lithosomes S and A (Ttsag),51 -121TSUQsc,Tesuque Fm coarse-grained lithosome S (Ttsc),52 -121TSUQsf,"Tesuque Fm, fine-grained lithosome S (Ttsf)",53 -122CHOC,Chamita and Ojo Caliente interlayered (Ttoc),54 -122CRTO,"Chama El Rito Formation (Tesuque member, Ttc)",55 -122OJOC,"Ojo Caliente Formation (Tesuque member, Tto)",56 -122PICR,Picuris Tuff,57 -122PPTS,Popotosa Formation,58 -122SNTFP,"Lower Santa Fe Group, piedmont facies",59 -123DTILSPRS,"Datil Group ignimbrites and lavas and Spears Group, interbedded",60 -123DTMGandbas,"Datil and Mogollon Group andesite, basaltic andesite, and basalt flows",61 -123DTMGign,Datil and Mogollon Group ignimbrites,62 -123DTMGrhydac,Datil and Mogollon Group rhyolite and dacite flows,63 -123ESPN,T Espinaso Formation (Te),64 -123GLST,T Galisteo Formation,65 -123PICS,T Picuris Formation (Tp),66 -123PICSc,"T Picuris Formation, basal conglomerate (Tpc)",67 -123PICSl,T lower Picuris Formation (Tpl),68 -123SPRSDTMGlava,"Spears Group and Datil-Mogollon intermediate-mafic lavas, interbedded",69 -123SPRSlower,"Spears Group, lower part; tuffaceous, gravelly debris and mud flows",70 -123SPRSmid_uppe,"Spears Group, middle-upper part; excludes Dog Spring Formation",71 -124BACA,Baca Formation,72 -124CBMN,Cub Mountain Formation,73 -124LLVS,Llaves Member of San Jose Formation,74 -124PSCN,Poison Canyon Formation,75 -124RGIN,Regina Member of San Jose Formation,76 -124SNJS,San Jose Formation,77 -124TPCS,TapicitosMember of San Jose Formation,78 -125NCMN,Nacimiento Formation,79 -125NCMNS,"Nacimiento Formation, Sandy Shale Facies",80 -125RTON,Raton Formation,81 -130CALDFLOOR,Caldera Floor bedrock S. of San Agustin Plains. Mostly DTILSPRS & Paleo.,82 -180TKSCC_Upper,"Tertiary-Cretaceous, Sanders Canyon, Cub Mtn. and upper Crevasse Canyon Fm",83 -180TKTR,"Tertiary-Cretaceous-Triassic, Baca, Crevasse Cyn, Gallup, Mancos, Dakota, T",84 -210CRCS,"Cretaceous System, undivided",85 -210GLUPC_Lower,K Gallup Sandstone and lower Crevasse Canyon Fm,86 -210HOSTD,K Hosta Dalton,87 -210MCDK,K Mancos/Dakota undivided,88 -210MNCS,"Mancos Shale, undivided",89 -210MNCSL,K Lower Mancos,90 -210MNCSU,K Upper Mancos,91 -211CLFHV,"Cliff House Sandstone, includes La Ventana Tongues in NW Sandoval Co.",92 -211CRLL,Carlile Shale,93 -211CRVC,Crevasse Canyon Formation of Mesaverde Group,94 -211DKOT,Dakota Sandstone or Formation,95 -211DLCO,Dilco Coal Member of Crevasse Canyon Formation of Mesaverde Group,96 -211DLTN,Dalton Sandstone Member of Crevasse Canyon Formation of Mesaverde Group,97 -211FRHS,Fort Hays Limestone Member of Niobrara Formation,98 -211FRLD,Fruitland Formation,99 -211FRMG,Farmington Sandstone Member of Kirtland Shale,100 -211GBSNC,Gibson Coal Member of Crevasse Canyon Formation of Mesaverde Group,101 -211GLLG,Gallego Sandstone Member of Gallup Sandstone,102 -211GLLP,Gallup Sandstone,103 -211GRRG,Greenhorn and Graneros Formations,104 -211GRRS,Graneros Shale,105 -211HOST,Hosta Tongue of Point Lookout Sandstone of Mesaverde Group,106 -211KRLD,Kirtland Shale,107 -211LWIS,Lewis Shale,108 -211MENF,Menefee Formation,109 -211MENFU,K Upper Menefee (above Harmon Sandstone),111 -211MVRD,Mesaverde Group,112 -211OJAM,Ojo Alamo Sandstone,113 -211PCCF,Pictured Cliffs Sandstone,114 -211PIRR,Pierre Shale,115 -211PNLK,Point Lookout Sandstone,116 -211SMKH,Smoky Hill Marl Member,117 -211TLLS,Twowells Sandstone Lentil of Pike of Dakota Sandstone,118 -212KTRP,"K Dakota Sandstone, Moenkopi Fm, Artesia Group",119 -217PRGR,Purgatoire Formation,120 -220ENRD,Entrada Sandstone,121 -220JURC,Jurassic undivided,122 -220NAVJ,Navajo Sandstone,123 -221BLFF,Bluff Sandstone of Morrison Formation,124 -221CSPG,Cow Springs Sandstone of Morrison Formation,125 -221ERADU,"Entrada Sandstone of San Rafael Group, Upper",126 -221MRSN,Morrison Formation,127 -221MRSN/BBSN,Brushy Basin Member of Morrison,128 -221MRSN/JCKP,Jackpile Sandstone Member of Morrison,129 -221MRSN/RCAP,Recapture Shale Member of Morrison,130 -221MRSN/WWCN,Westwater Canyon Member of Morrison,131 -221SLWS,Salt Wash Sandstone Member of Morrison Formation,132 -221SMVL,Summerville Formation of San Rafael Group,133 -221TDLT,J Todilto,134 -221WSRC,Westwater Canyon Sandstone Member of Morrison Formation,135 -221ZUNIS,Zuni Sandstone,136 -231AGZC,Tr Agua Zarca,137 -231AGZCU,Tr Upper Agua Zarca,138 -231CHNL,Chinle Formation,139 -231CORR,Correo Sandstone Member of Chinle Formation,140 -231DCKM,Dockum Group,141 -231PFDF,Tr Petrified Forest,142 -231PFDFL,Tr Lower Petrified Forest (below middle sandstone),143 -231PFDFM,Tr Middle Petrified Forest sandstone,144 -231PFDFU,Tr Upper Petrified Forest (above middle sandstone),145 -231RCKP,Rock Point Member of Wingate Sandstone,146 -231SNRS,Santa Rosa Sandstone,147 -231SNSL,Sonsela Sandstone Bed of Petrified Forest Member of Chinle Formation,148 -231SRMP,Shinarump Member of Chinle Formation,149 -231WNGT,Wingate Sandstone,150 -260SNAN,P San Andres,151 -260SNAN_lower,Lower San Andres Formation,152 -261SNGL,P San Andres - Glorieta Sandstone in Rio Bonito member,153 -300YESO,P Yeso,154 -300YESO_lower,Lower Yeso Formation,155 -300YESO_upper,Upper Yeso Formation,156 -310ABO,P Abo,157 -310DCLL,De Chelly Sandstone Member of Cutler Formation,158 -310GLOR,Glorieta Sandstone Member of San Andres Formation (of Manzano Group),159 -310MBLC,Meseta Blanca Sandstone Member of Yeso Formation,160 -310TRRS,Torres Member of Yeso Formation,161 -310YESO,Yeso Formation,162 -310YESOG,"Yeso Formation, Manzono Group",163 -312CSTL,Castile Formation,164 -312RSLR,Rustler Formation,165 -313ARTS,Artesia Group,166 -313BLCN,Bell Canyon Formation,167 -313BRUC,Brushy Canyon Formation of Delaware Mountain Group,168 -313CKBF,Chalk Bluff Formation,169 -313CLBD,Carlsbad Limestone,170 -313CPTN,Capitan Limestone,171 -313GDLP,Guadalupian Series,172 -313GOSP,Goat Seep Dolomite,173 -313SADG,San Andres Limestone and Glorieta Sandstone,174 -313SADR,"San Andres Limestone, undivided",175 -313TNSL,Tansill Formation,176 -313YATS,"Yates Formation, Guadalupe Group",177 -315LABR,P Laborcita (Bursum),178 -315YESOABO,Alamosa Creek and San Agustin Plains area - Yeso and Abo Formations,179 -318ABO,P Abo,180 -318BSPG,Bone Spring Limestone,181 -318JOYT,Joyita Sandstone Member of Yeso Formation,182 -318YESO,Yeso Formation,183 -319BRSM,Bursum Formation and Equivalent Rocks,184 -320HLDR,Penn Holder,185 -320PENN,Pennsylvanian undivided,186 -320SNDI,Sandia Formation,187 -321SGDC,Sangre de Cristo Formation,188 -322BEMN,Penn Beeman,189 -325GBLR,Penn Gobbler,190 -325MDER,"Madera Limestone, undivided",191 -325MDERL,Penn Lower Madera,192 -325MDERU,Penn Upper Madera,193 -325SAND,Penn Sandia,194 -326MGDL,Magdalena Group,195 -340EPRS,Espiritu Santo Formation,196 -350PZBA,Alamosa Creek and San Agustin Plains area - Paleozoic strata beneath Abo Fm,197 -350PZBB,Tul Basin area - Paleozoic strata below Bursum Fm,198 -400EMBD,Embudo Granite (undifferentiated PreCambrian near Santa Fe),199 -400PCMB,Precambrian Erathem,200 -400PREC,undifferentiated PreCambrian crystalline rocks (X),201 -400PRECintr,PreCambrian crystalline rocks and local Tertiary intrusives,202 -400PRST,Priest Granite,203 -400TUSS,Tusas Granite,204 -410PRCG,PreCambrian granite (Xg),205 -410PRCGf,"PreCambrian granite, fractured (Xgf)",206 -410PRCQ,PreCambrian quartzite (Xq),207 -410PRCQf,"PreCambrian quartzite, fractured (Xqf)",208 -121GILA,Gila Conglomerate (group),209 -312DYLK,Dewey Lake Redbeds,210 -120WMVL,Wimsattville Formation,211 -313GRBG,Grayburg Formation of Artesia Group,212 -318ABOL,Abo Sandstone (Lower Tongue),213 -318ABOU,Abo Sandstone (Upper Tongue),214 -112SNTFU,"Santa Fe Group, Upper Part",215 -310FRNR,Forty-Niner Member of Rustler Formation,216 -312OCHO,Ochoan Series,217 -313AZOT,Azotea Tongue of Seven Rivers Formation,218 -313QUEN,Queen Formation,219 -319HUCO,Hueco Limestone,220 -313SVRV,Seven Rivers Formation,221 -313CABD,Carlsbad Group,222 -320GRMS,Gray Mesa Member of Madera Formation,223 -211CLRDH,Colorado Shale,224 -120BRLM,Bearwallow Mountain Andesite,225 -122RUBO,Rubio Peak Formation,226 -313SADRL,"San Andres Limestone, Lower Cherty Member",227 -313SADRU,"San Andres Limestone, Upper Clastic Member",228 -313BRNL,Bernal Formation of Artesia Group,229 -318CPDR,Chupadera Formation,230 -121BDHC,Bidahochi Formation,231 -313SADY,"San Andres Limestone and Yeso Formation, undivided",232 -221SRFLL,"San Rafael Group, Lower Part",233 -221BLUF,Bluff Sandstone of Morrison Formation,234 -221COSP,Cow Springs Sandstone of Morrison Formation,235 -317ABYS,"Abo and Yeso, undifferentiated",236 -221BRSB,Brushy Basin Shale Member of Morrison Formation,237 -310SYDR,San Ysidro Member of Yeso Formation,238 -400SDVL,Sandoval Granite,239 -221SRFL,San Rafael Group,240 -310SGRC,Sangre de Cristo Formation,241 -231TCVS,Tecovas Formation of Dockum Group,242 -211DCRS,D-Cross Tongue of Mancos Shale of Mesaverde Group,243 -211ALSN,Allison Member of Menefee Formation of Mesaverde Group,244 -211LVNN,La Ventana Tongue of Cliff House Sandstone,245 -211MORD,Madrid Formation,246 -210PRMD,Pyramid Shale,247 -124ANMS,Animas Formation,248 -211NBRR,Niobrara Formation,249 -111ALVM,Holocene Alluvium,250 -122SNTFL,"Santa Fe Group, Lower Part",251 -111CPLN,Capulin Basalts,252 -120CRSN,Carson Conflomerate,253 -111CRMS,Covered/Reclaimed Mine Spoil,254 -111CRMSA,Covered/Reclaimed Mine Spoil and Ash,255 -111SPOL,Spoil,256 -110TURT,Tuerto Gravel of Santa Fe Group,257 -221RCPR,Recapture Shale Member of Morrison Formation,258 -320BLNG,Bullington Member of Magdalena Formation,259 -112ANCHsr,"Upper Santa Fe Group, Ancha Formation & ancestral Santa Fe river deposits",260 -121TSUQae,Tesuque Fm Lithosomes A and E,261 -230TRSC,Triassic undifferentiated,262 -122TSUQdx,"Tesuque Fm, Dixon member (Ttd)",263 -123PICSu,T upper Picuris Formation (Tpu),264 -123PICSm,T middle Picuris Formation (Tpm),265 -123PICSmc,T middle conglomerate Picuris Formation (Tpmc),266 -120VBVC,Tertiary volcanic breccia/volcaniclastic conglomerate,267 -120VCSS,Tertiary volcaniclastic sandstone,268 -124DMDT,Diamond Tail Formation,269 -325ALMT,Penn Alamitos Formation,270 -400SAND,Sandia Granite,271 -318VCPK,Victorio Peak Limestone,272 -318BSVP,Bone Spring and Victorio Peak Limestones,273 -100ALVM,Alluvium,274 -310PRMN,Permian System,275 -110AVPS,Alluvium and Permian System,276 -313CRCX,Capitan Reef Complex and Associated Limestones,277 -112SLBL,Salt Bolson,278 -112SBCRC,Salt Bolson and Capitan Reef Complex,279 -313CRDM,Capitan Reef Complex - Delaware Mountain Group,280 -112SBDM,Salt Bolson and Delaware Mountain Group,281 -120BLSN,Bolson Deposits,282 -112SBCR,Salt Bolson and Cretaceous Rocks,283 -112HCBL,Hueco Bolson,284 -120IVIG,Intrusive Rocks,285 -112RLBL,Red Light Draw Bolson,286 -112EFBL,Eagle Flat Bolson,287 -112GRBL,Green River Bolson,288 -123SAND,Sanders Canyon Formation,289 -210MRNH,Moreno Hill Formation,291 -320ALMT,Alamito Shale,292 -313DLRM,Delaware Mountain Group,293 -300PLZC,Paleozoic Erathem,294 -122SPRS,Spears Member of Datil Formation,295 -110AVTV,Alluvium and Tertiary Volcanics,296 -313DMBS,Delaware Mountain Group - Bone Spring Limestone,297 -120ERSV,Tertiary extrusives,298 diff --git a/transfers/data/lookup_tables/LU_LevelStatus.csv b/transfers/data/lookup_tables/LU_LevelStatus.csv deleted file mode 100644 index 435083a4a..000000000 --- a/transfers/data/lookup_tables/LU_LevelStatus.csv +++ /dev/null @@ -1,24 +0,0 @@ -CODE,MEANING,id,WebDataEntryVisible -A,Water level affected by atmospheric pressure,1,0 -C,Water level was frozen (no level recorded).,2,1 -D,Site was dry,3,1 -E,Site was flowing recently.,4,1 -F,Site was flowing. Water level or head couldn't be measured w/out additional equipment.,5,1 -G,Nearby site that taps the same aquifer was flowing.,6,1 -H,Nearby site that taps the same aquifer had been flowing recently.,7,1 -I,Recharge water was being injected into the aquifer at this site.,8,0 -J,Recharge water was being injected into nearby site that taps the same aquifer.,9,0 -K,Water was cascading down the inside of the well.,10,1 -L,Water level was affected by brackish or saline water.,11,0 -M,Well was not in hydraulic contact w/formation (from source other than defined in USGS C714 or C93).,12,0 -N,Measurement was discontinued (no level recorded).,13,0 -O,Obstruction was encountered in the well (no level recorded),14,1 -P,Site was being pumped,15,1 -R,Site was pumped recently,16,1 -S,Nearby site that taps the same aquifer was being pumped,17,1 -T,Nearby site that taps the same aquifer was pumped recently,18,1 -V,Foreign substance present on the water surface,19,1 -W,Well was destroyed (no subsequent water levels should be recorded),20,1 -X,Water level affected by stage in nearby surface-water site,21,1 -Z,Other conditions exist that would affect the level (remarks),22,1 -AA,Water level not affected by status,24,0 diff --git a/transfers/data/lookup_tables/LU_Lithology.csv b/transfers/data/lookup_tables/LU_Lithology.csv deleted file mode 100644 index 970c3dda7..000000000 --- a/transfers/data/lookup_tables/LU_Lithology.csv +++ /dev/null @@ -1,77 +0,0 @@ -TERM,ABBREVIATION,id -Alluvium,ALVM,1 -Anhydrite,ANDR,2 -Arkose,ARKS,3 -Boulders,BLDR,4 -"Boulders, silt and clay",BLSC,5 -Boulders and sand,BLSD,6 -Bentonite,BNTN,7 -Breccia,BRCC,8 -Basalt,BSLT,9 -Conglomerate,CGLM,10 -Chalk,CHLK,11 -Chert,CHRT,12 -Clay,CLAY,13 -Caliche,CLCH,14 -Calcite,CLCT,15 -"Clay, some sand",CLSD,16 -Claystone,CLSN,17 -Coal,COAL,18 -Cobbles,COBB,19 -"Cobbles, silt and clay",COSC,20 -Cobbles and sand,COSD,21 -Dolomite,DLMT,22 -Dolomite and shale,DLSH,23 -Evaporite,EVPR,24 -Gneiss,GNSS,25 -Gypsum,GPSM,26 -Graywacke,GRCK,27 -Gravel and clay,GRCL,28 -"Gravel, cemented",GRCM,29 -"Gravel, sand and silt",GRDS,30 -"Granite, gneiss",GRGN,31 -Granite,GRNT,32 -"Gravel, silt and clay",GRSC,33 -Gravel,GRVL,34 -Igneous undifferentiated,IGNS,35 -Lignite,LGNT,36 -Limestone and dolomite,LMDM,37 -Limestone and shale,LMSH,38 -Limestone,LMSN,39 -Marl,MARL,40 -Mudstone,MDSN,41 -Metamorphic undifferentiated,MMPC,42 -Marlstone,MRLS,43 -No Recovery,NR,44 -Peat,PEAT,45 -Quartzite,QRTZ,46 -Rhyolite,RYLT,47 -Sand,SAND,48 -Schist,SCST,49 -Sand and clay,SDCL,50 -Sand and gravel,SDGL,51 -Sandstone and shale,SDSL,52 -Sand and silt,SDST,53 -"Sand, gravel and clay",SGVC,54 -Shale,SHLE,55 -Silt,SILT,56 -Siltstone and shale,SLSH,57 -Siltstone,SLSN,58 -Slate,SLTE,59 -"Sand, some clay",SNCL,60 -Sandstone,SNDS,61 -Silt and clay,STCL,62 -Travertine,TRVR,63 -Tuff,TUFF,64 -Volcanic undifferentiated,VLCC,65 -"Clay, yellow",CLYE,66 -"Clay, red",CLRD,67 -Surficial sediment,SUSE,68 -"Limestone and sandstone, interbedded",LMSD,70 -Gravel and boulders,GRBL,71 -"Sand, silt and gravel",SSTG,72 -"Sand, gravel, silt and clay",SGSC,73 -Andesite,ANDS,74 -"Ignesous, intrusive, undifferentiated",IGIN,75 -"Limestone, sandstone and shale",LMSS,76 -"Sand, silt and clay",SDCS,77 diff --git a/transfers/data/lookup_tables/LU_MajorAnalyte.csv b/transfers/data/lookup_tables/LU_MajorAnalyte.csv deleted file mode 100644 index f4ae20664..000000000 --- a/transfers/data/lookup_tables/LU_MajorAnalyte.csv +++ /dev/null @@ -1,23 +0,0 @@ -CODE,MEANING,id -ALK,"Alkalinity, Total",1 -Ca,Calcium,2 -Ca(total),"Calcium, total, unfiltered",3 -Cl,Chloride,4 -CO3,Carbonate,5 -CONDLAB,"Conductivity, laboratory (µS)",6 -HCO3,Bicarbonate,7 -HRD,Hardness (CaCO3),8 -IONBAL,Ion Balance,9 -K,Potassium,10 -K(total),"Potassium, total, unfiltered",11 -Mg,Magnesium,12 -Mg(total),"Magnesium, total, unfiltered",13 -Na,Sodium,14 -Na(total),"Sodium, total, unfiltered",15 -Na+K,Sodium and Potassium combined,16 -OH,Alkalinity as OH-,17 -pHL,"pH,laboratory",18 -SO4,Sulfate,19 -TAn,Total Anions,20 -TCat,Total Cations,21 -TDS,Total Dissolved Solids,22 diff --git a/transfers/data/lookup_tables/LU_MeasurementMethod.csv b/transfers/data/lookup_tables/LU_MeasurementMethod.csv deleted file mode 100644 index 981ece641..000000000 --- a/transfers/data/lookup_tables/LU_MeasurementMethod.csv +++ /dev/null @@ -1,25 +0,0 @@ -CODE,MEANING,id,WebDataEntryVisible -A,Airline measurement,1,1 -B,Analog or graphic recorder,2,0 -C,Calibrated airline measurement,3,0 -D,Differential GPS; especially applicable to surface expression of ground water,4,0 -E,Estimated,5,0 -F,Transducer,6,0 -G,Pressure-gage measurement,7,1 -H,Calibrated pressure-gage measurement,8,0 -L,Interpreted from geophysical logs,9,0 -M,Manometer,10,0 -N,Non-recording gage,11,0 -O,"Observed (required for F, N, and W water level status)",12,0 -P,Sonic water level meter (acoustic pulse),13,1 -R,"Reported, method not known",14,0 -S,Steel-tape measurement,15,1 -T,Electric tape measurement (E-probe),16,1 -U,Unknown (for legacy data only; not for new data entry),17,0 -V,Calibrated electric tape; accuracy of equipment has been checked,18,0 -W,Calibrated electric cable,19,0 -X,Uncalibrated electric cable,20,0 -Z,Other (explain in notes),21,0 -K,Continuous acoustic sounder,22,0 -ZZ,Measurement not attempted,23,1 -0,null placeholder,24,0 diff --git a/transfers/data/lookup_tables/LU_MeasuringAgency.csv b/transfers/data/lookup_tables/LU_MeasuringAgency.csv deleted file mode 100644 index f142a02b1..000000000 --- a/transfers/data/lookup_tables/LU_MeasuringAgency.csv +++ /dev/null @@ -1,19 +0,0 @@ -Agency,Description,id -USGS,US Geological Survey,1 -NMOSE,New Mexico Office of the State Engineer,2 -NMBGMR,New Mexico Bureau of Geology & Mineral Resources,3 -Bernalillo Cty,Bernalillo County,4 -BLM,Bureau of Land Management,5 -SFC,Santa Fe County,6 -NESWCD,Northeast Soil & Water Conservation District,7 -NMED,New Mexico Environment Dept.,8 -NMISC,New Mexico Interstate Stream Commission,9 -PVACD,Pecos Valley Artesian Conservancy District,10 -TSWCD,Taos Soil & Water Conservation District,11 -Bayard,Bayard Municipal Water,12 -OSWCD,Otero Soil & Water Conservation District,13 -SNL,Sandia National Laboratories,14 -USFS,US Forest Service,15 -TWDB,Texas Water Development Board,16 -NMT,New Mexico Tech,17 -NPS,National Park Service,18 diff --git a/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv b/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv deleted file mode 100644 index 6fd3653e3..000000000 --- a/transfers/data/lookup_tables/LU_MinorTraceAnalyte.csv +++ /dev/null @@ -1,93 +0,0 @@ -CODE,MEANING,id -3H,Tritium ,1 -3H:3He Age,Age of Water using dissolved gases,2 -Ag,Silver,3 -Ag(total),"Silver, total, unfiltered",4 -Al,Aluminum,5 -Al(total),"Aluminum, total, unfiltered",6 -As,Arsenic,7 -As(total),"Arsenic, total, unfiltered",8 -B,Boron,9 -B(total),"Boron, total, unfiltered",10 -Ba,Barium,11 -Ba(total),"Barium, total, unfiltered",12 -Be,Beryllium,13 -Be(total),"Beryllium, total, unfiltered",14 -Br,Bromide,15 -C13r,13C:12C ratio ,16 -C14,"14C content, pmc",17 -C14_years,Uncorrected C14 age,18 -Cd,Cadmium,19 -Cd(total),"Cadmium, total, unfiltered",20 -CFC11,Chlorofluorocarbon-11 avg age,21 -CFC113,Chlorofluorocarbon-113 avg age,22 -CFC113_12,Chlorofluorocarbon-113/12 avg RATIO age,23 -CFC12,Chlorofluorocarbon-12 avg age,24 -Co,Cobalt,25 -Co(total),"Cobalt, total, unfiltered",26 -Cr,Chromium,27 -Cr(total),"Chromium, total, unfiltered",28 -Cu,Copper,29 -Cu(total),"Copper, total, unfiltered",30 -d18O-SO4,delta O18 sulfate,31 -d34S-SO4,Sulfate 34 isotope ratio,32 -F,Fluoride,33 -Fe,Iron,34 -Fe(total),"Iron, total, unfiltered",35 -H2r,Deuterium:Hydrogen ratio ,36 -Hg,Mercury,37 -Hg(total),"Mercury, total, unfiltered",38 -Li,Lithium,39 -Li(total),"Lithium, total, unfiltered",40 -Mn,Manganese,41 -Mn(total),"Manganese, total, unfiltered",42 -Mo,Molybdenum,43 -Mo(total),"Molybdenum, total, unfiltered",44 -Ni,Nickel,45 -Ni(total),"Nickel, total, unfiltered",46 -NO2,Nitrite (as NO2),47 -NO2(N),Nitrite (as N),48 -NO3,Nitrate (as NO3),49 -NO3(N),Nitrate (as N),50 -O18r,18O:16O ratio ,51 -Pb,Lead,52 -Pb(total),"Lead, total, unfiltered",53 -PO4,Phosphate,54 -Sb,Antimony,55 -Sb(total),"Antimony, total, unfiltered",56 -Se,Selenium,57 -Se(total),"Selenium, total, unfiltered",58 -Sf6,Sulfur hexafluoride,59 -Si,Silicon,60 -Si(total),"Silicon, total, unfiltered",61 -SiO2,Silica ,62 -Sn,Tin,63 -Sn(total),"Tin, total, unfiltered",64 -Sr,Strontium,65 -Sr(total),"Strontium, total, unfiltered",66 -Sr87:Sr86,Strontium 87:86 ratio,67 -Th,Thorium,68 -Th(total),"Thorium, total, unfiltered",69 -Ti,Titanium,70 -Ti(total),"Titanium, total, unfiltered",71 -Tl,Thallium,72 -Tl(total),"Thallium, total, unfiltered",73 -U,"Uranium (total, by ICP-MS)",74 -U(total),"Uranium, total, unfiltered",75 -V,Vanadium,76 -V(total),"Vanadium, total, unfiltered",77 -Zn,Zinc,78 -Zn(total),"Zinc, total, unfiltered",79 -C14_corrected,Corrected C14 in years,80 -AsIII,Arsenite (arsenic species),81 -AsV,Arsenate (arsenic species),82 -CN6,Cyanide,83 -ERT,Estimated recharge temperature,84 -H2S,Hydrogen sulfide,85 -NH3,Ammonia,86 -NH4,Ammonium,87 -TN,Total nitrogen,88 -TKN,Total Kjeldahl nitrogen,89 -DOC,Dissolved organic carbon,90 -TOC,Total organic carbon,91 -d13C-DIC,delta C13 of dissolved inorganic carbon,93 diff --git a/transfers/data/lookup_tables/LU_MonitoringStatus.csv b/transfers/data/lookup_tables/LU_MonitoringStatus.csv deleted file mode 100644 index 17c1206b2..000000000 --- a/transfers/data/lookup_tables/LU_MonitoringStatus.csv +++ /dev/null @@ -1,17 +0,0 @@ -CODE,MEANING,id -6,Monitor every six months,1 -A,Annual water level,2 -B,Monitoring bi-monthly,3 -C,Monitoring complete,4 -D,Datalogger installed,5 -L,Monitor every 10 years (long-term monitor),6 -M,Monitor monthly,7 -Q,Sampling complete,8 -R,Reported to NMBGMR bimonthly,9 -S,Sample well,10 -X,Water level cannot be measured,11 -P,Repeat sampling,12 -W,Wellntel device,13 -N,Bi-annual (every other year),14 -I,Inactive,15 -H,Data share,16 diff --git a/transfers/data/lookup_tables/LU_SampleType.csv b/transfers/data/lookup_tables/LU_SampleType.csv deleted file mode 100644 index 698988f9d..000000000 --- a/transfers/data/lookup_tables/LU_SampleType.csv +++ /dev/null @@ -1,12 +0,0 @@ -CODE,MEANING,id -BA,Background,1 -EB,Equipment blank,2 -FB,Field blank,3 -FD,Field duplicate,4 -FP,Field parameters only,5 -PR,Precipitation,6 -RE,Repeat sample,7 -SD,Standard field sample,8 -SR,Soil or Rock sample,9 -TB,Trip blank,10 -SB,Source water blank,11 diff --git a/transfers/data/lookup_tables/LU_SiteType.csv b/transfers/data/lookup_tables/LU_SiteType.csv deleted file mode 100644 index e70fc2b09..000000000 --- a/transfers/data/lookup_tables/LU_SiteType.csv +++ /dev/null @@ -1,12 +0,0 @@ -CODE,MEANING,id -D,"Diversion of surface water, etc",1 -ES,Ephemeral stream,2 -GW,Groundwater other than spring (well),3 -L,"Lake, pond or reservoir",4 -M,"Meteorological (rain, snow)",5 -O,Outfall of wastewater or return flow,6 -OT,Other,7 -PS,Perennial stream,8 -R,Rock sample location,9 -S,Soil gas sample location,10 -SP,Spring,11 diff --git a/transfers/data/lookup_tables/LU_Status.csv b/transfers/data/lookup_tables/LU_Status.csv deleted file mode 100644 index f596a2476..000000000 --- a/transfers/data/lookup_tables/LU_Status.csv +++ /dev/null @@ -1,6 +0,0 @@ -CODE,MEANING,id -A,Abandoned,1 -C,"Active, pumping well",2 -D,"Destroyed, exists but not usable",3 -I,"Inactive, exists but not used",4 -U,Unknown,5 From 8bedd145dae0daf34876c54ae76e366767d2436d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 09:11:51 -0600 Subject: [PATCH 13/33] refactor: use variable for WGS84 SRID throughout --- constants.py | 1 + db/group.py | 3 ++- db/location.py | 11 ++++++----- services/util.py | 8 +++++--- tests/test_collabnet.py | 11 +++-------- tests/test_geospatial.py | 5 +++-- transfers/util.py | 3 ++- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/constants.py b/constants.py index 2944f549f..93179ddb1 100644 --- a/constants.py +++ b/constants.py @@ -15,4 +15,5 @@ # =============================================================================== SRID_WGS84 = 4326 +SRID_UTM_ZONE_13N = 26913 # ============= EOF ============================================= diff --git a/db/group.py b/db/group.py index 6dbddb89a..66f7a717b 100644 --- a/db/group.py +++ b/db/group.py @@ -20,6 +20,7 @@ from sqlalchemy.orm import relationship, Mapped from sqlalchemy.testing.schema import mapped_column +from constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin @@ -28,7 +29,7 @@ class Group(Base, AutoBaseMixin, ReleaseMixin): description: Mapped[str] = mapped_column(String(255), nullable=True) name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) project_area: Mapped[Optional[WKBElement]] = mapped_column( - Geometry(geometry_type="MULTIPOLYGON", srid=4326, spatial_index=True) + Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True) ) # Foreign Keys diff --git a/db/location.py b/db/location.py index f54d7a1f7..ed5da2703 100644 --- a/db/location.py +++ b/db/location.py @@ -21,8 +21,6 @@ from uuid import UUID from sqlalchemy import ( - Column, - Integer, String, ForeignKey, DateTime, @@ -32,6 +30,7 @@ from sqlalchemy.orm import relationship, Mapped, mapped_column from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy +from constants import SRID_WGS84 from db.base import Base, AutoBaseMixin, ReleaseMixin from db.lexicon import lexicon_term @@ -45,7 +44,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): description: Mapped[str] = mapped_column name: Mapped[str] = mapped_column(String(255), nullable=True) point: Mapped[WKBElement] = mapped_column( - Geometry(geometry_type="POINTZ", srid=4326, spatial_index=True) + Geometry(geometry_type="POINTZ", srid=SRID_WGS84, spatial_index=True) ) state: Mapped[str] = lexicon_term(nullable=True, default="New Mexico") @@ -65,7 +64,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): ) # --- Proxy Definitions --- - things: AssociationProxy[list["Thing"]] = association_proxy( + things: AssociationProxy[list["Thing"]] = association_proxy( # noqa: F821 "thing_associations", "thing" ) @@ -95,7 +94,9 @@ class LocationThingAssociation(Base, AutoBaseMixin): # --- Relationship Definitions --- location: Mapped["Location"] = relationship(back_populates="thing_associations") - thing: Mapped["Thing"] = relationship(back_populates="location_associations") + thing: Mapped["Thing"] = relationship( # noqa: F821 + back_populates="location_associations" + ) # ============= EOF ============================================= diff --git a/services/util.py b/services/util.py index 274507b95..f134ecad1 100644 --- a/services/util.py +++ b/services/util.py @@ -2,6 +2,8 @@ import pyproj import httpx +from constants import SRID_WGS84 + TRANSFORMERS = {} @@ -31,7 +33,7 @@ def get_tiger_data( "where": "1=1", "geometry": f"{lon},{lat}", "geometryType": "esriGeometryPoint", - "inSR": "4326", + "inSR": f"{SRID_WGS84}", "spatialRel": "esriSpatialRelIntersects", "outFields": outfields, "returnGeometry": "false", @@ -65,7 +67,7 @@ def get_quad_name_from_point(lon: float, lat: float) -> str: "f": "json", "geometry": f"{lon},{lat}", "geometryType": "esriGeometryPoint", - "inSR": "4326", + "inSR": f"{SRID_WGS84}", "spatialRel": "esriSpatialRelIntersects", "outFields": "CELL_NAME,CELL_MAPCODE", "returnGeometry": "false", @@ -88,7 +90,7 @@ def get_epqs_elevation_from_point(lon: float, lat: float) -> float: "x": lon, "y": lat, "units": "Meters", - "wkid": "4326", + "wkid": f"{SRID_WGS84}", "includeDate": False, } diff --git a/tests/test_collabnet.py b/tests/test_collabnet.py index 2a8ba0d7f..96712e081 100644 --- a/tests/test_collabnet.py +++ b/tests/test_collabnet.py @@ -13,17 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime - import pytest +from constants import SRID_WGS84 from db import Location -from db.engine import get_db_session, session_ctx +from db.engine import session_ctx -# from db import get_db_session, database_sessionmaker -# from db.base import Location -# from db.thing.well import WellThing -# from db.collabnet import CollaborativeNetworkWell from services.thing_helper import add_thing from tests import client @@ -31,7 +26,7 @@ @pytest.fixture(scope="function") def well(): with session_ctx() as session: - loc = Location(point="SRID=4326;POINT(0 0)") + loc = Location(point=f"SRID={SRID_WGS84};POINT(0 0)") session.add(loc) session.commit() diff --git a/tests/test_geospatial.py b/tests/test_geospatial.py index 5759a87d4..52e1bf0f2 100644 --- a/tests/test_geospatial.py +++ b/tests/test_geospatial.py @@ -17,6 +17,7 @@ import pytest from main import app +from constants import SRID_WGS84 from core.dependencies import ( admin_function, editor_function, @@ -76,11 +77,11 @@ def populate(): loc1 = Location( name="Test Location 1", - point=geofunc.ST_GeomFromText("POINT(10.1 10.1 0)", srid=4326), + point=geofunc.ST_GeomFromText("POINT(10.1 10.1 0)", srid=SRID_WGS84), ) loc2 = Location( name="Test Location 2", - point=geofunc.ST_GeomFromText("POINT(20 20 0)", srid=4326), + point=geofunc.ST_GeomFromText("POINT(20 20 0)", srid=SRID_WGS84), ) session.add(loc1) session.add(loc2) diff --git a/transfers/util.py b/transfers/util.py index c6090f262..2c4e899a9 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -23,6 +23,7 @@ from sqlalchemy.orm import Session import pandas as pd +from constants import SRID_WGS84, SRID_UTM_ZONE_13N from db import Thing, Location from services.util import ( transform_srid, @@ -100,7 +101,7 @@ def make_location(row: pd.Series) -> Location: # Convert the point to a WGS84 coordinate system transformed_point = transform_srid( - point, source_srid=26913, target_srid=4326 # WGS84 SRID + point, source_srid=SRID_UTM_ZONE_13N, target_srid=SRID_WGS84 ) state = get_state_from_point(transformed_point.x, transformed_point.y) From 9251bb869f3908e4f324988b6b4bab7a3a55b1dd Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 09:57:32 -0600 Subject: [PATCH 14/33] WIP: apply LU-lexicon mapper --- transfers/transfer.py | 7 ++++--- transfers/util.py | 10 ++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 9df2c94e5..7f387d4e4 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -59,14 +59,15 @@ def erase_and_initalize(session: Session) -> None: logger.info(f"Done initializing sensors. {elapsed_time:0.2f}s") -def message(msg, pad=10): +def message(msg, pad=10, new_line_at_top=True): pad = "*" * pad - logger.info("") + if new_line_at_top: + logger.info("") logger.info(f"{pad} {msg} {pad}") def main_transfer(): - logger.info("Starting transfer") + message("STARTING TRANSFER", new_line_at_top=False) init = True diff --git a/transfers/util.py b/transfers/util.py index b7cdc93de..ebc507d51 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -149,7 +149,7 @@ def make_location(row: pd.Series) -> Location: ) z = get_epqs_elevation_from_point(transformed_point.x, transformed_point.y) - setattr(point, z, z) + point_with_z = Point(point.x, point.y, z) # TODO: determine correct created_at value # created_at = row.DateCreated @@ -159,16 +159,14 @@ def make_location(row: pd.Series) -> Location: nma_pk_location=row.LocationId, # TODO: determine if PointID should map to location.name or thing.name or if the Location table needs a name field at all. name=row.PointID, - point=transformed_point.wkt, + point=point_with_z.wkt, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, - # TODO: map code to meaning since meaning is used as the lexicon term - # elevation_method=row.AltitudeMethod, + elevation_method=lu_to_lexicon_map[row.AltitudeMethod], # created_at=created_at, # TODO: row.CoordinateAccuracy is not a float # coordinate_accuracy=row.CoordinateAccuracy, - # TODO: map code to meaning since meaning is used as the lexicon term - # coordinate_method=row.CoordinateMethod, + coordinate_method=lu_to_lexicon_map[row.CoordinateMethod], nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, state=state, From 965f563a8cf8ecac580432cecf3c73b86933ee78 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 10:13:14 -0600 Subject: [PATCH 15/33] feat: prepend mapper key with LU table name for uniqueness --- transfers/util.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index 59c420b92..fff89b753 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -311,20 +311,14 @@ def make_lu_to_lexicon_mapper(): code = row.CODE meaning = row.MEANING - mappers.update({code: meaning}) + mappers.update({f"{lu_table}:{code}": meaning}) return mappers lu_to_lexicon_map = make_lu_to_lexicon_mapper() + if __name__ == "__main__": - # quad = get_quad_name_from_point(-106.5, 34.2) - # print(quad) - # state = get_state_from_point(-106.5, 34.2) - # print(state) - # county = get_county_from_point(-106.5, 34.2) - # print(county) - z = get_epqs_elevation(-106.5, 34.2) - print(z) + print(lu_to_lexicon_map) # ============= EOF ============================================= From b3cc66f4f1fbe68fabe764c316e4e52010cff934 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 12:56:09 -0600 Subject: [PATCH 16/33] refactor: remove name from location until future decision is made --- db/location.py | 2 +- schemas/location.py | 6 +++--- tests/conftest.py | 4 ++-- tests/test_geospatial.py | 4 ++-- tests/test_location.py | 19 +++++++++++-------- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/db/location.py b/db/location.py index ed5da2703..bc16ad8eb 100644 --- a/db/location.py +++ b/db/location.py @@ -42,7 +42,7 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): String(36), nullable=True, unique=True ) description: Mapped[str] = mapped_column - name: Mapped[str] = mapped_column(String(255), nullable=True) + # name: Mapped[str] = mapped_column(String(255), nullable=True) point: Mapped[WKBElement] = mapped_column( Geometry(geometry_type="POINTZ", srid=SRID_WGS84, spatial_index=True) ) diff --git a/schemas/location.py b/schemas/location.py index d98235108..af26edc0a 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -34,7 +34,7 @@ class CreateLocation(BaseCreateModel): Schema for creating a sample location. """ - name: str | None = None + # name: str | None = None notes: str | None = None point: str # point is required and should be in WKT format release_status: str | None = "draft" @@ -64,7 +64,7 @@ class LocationResponse(BaseResponseModel): Response schema for sample location details. """ - name: str | None + # name: str | None notes: str | None point: str release_status: str | None @@ -103,7 +103,7 @@ class UpdateLocation(BaseUpdateModel): Schema for updating a location. """ - name: str | None = None + # name: str | None = None notes: str | None = None point: str | None = None release_status: str | None = None diff --git a/tests/conftest.py b/tests/conftest.py index aedc157d6..3489334d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ def location(): with session_ctx() as session: loc = Location( - name="first location", + # name="first location", notes="these are some test notes", point="POINT(-107.949533 33.809665 2464.9)", release_status="draft", @@ -33,7 +33,7 @@ def location(): def second_location(): with session_ctx() as session: location = Location( - name="second location", + # name="second location", point="POINT (10.2 10.2 0)", release_status="draft", ) diff --git a/tests/test_geospatial.py b/tests/test_geospatial.py index 52e1bf0f2..779e6a3c3 100644 --- a/tests/test_geospatial.py +++ b/tests/test_geospatial.py @@ -76,11 +76,11 @@ def populate(): session.commit() loc1 = Location( - name="Test Location 1", + # name="Test Location 1", point=geofunc.ST_GeomFromText("POINT(10.1 10.1 0)", srid=SRID_WGS84), ) loc2 = Location( - name="Test Location 2", + # name="Test Location 2", point=geofunc.ST_GeomFromText("POINT(20 20 0)", srid=SRID_WGS84), ) session.add(loc1) diff --git a/tests/test_location.py b/tests/test_location.py index fc3811c62..e2f849ac1 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -42,7 +42,7 @@ def override_dependencies_fixture(): def test_add_location(): payload = { - "name": "test location", + # "name": "test location", "notes": "these are some test notes", "point": "POINT Z (-106.607784 35.118924 1558.8)", "release_status": "draft", @@ -57,7 +57,7 @@ def test_add_location(): data = response.json() assert "id" in data assert "created_at" in data - assert data["name"] == payload["name"] + # assert data["name"] == payload["name"] assert data["notes"] == payload["notes"] assert data["point"] == payload["point"] assert data["release_status"] == payload["release_status"] @@ -78,7 +78,7 @@ def test_add_location(): def test_update_location(location): payload = { - "name": "patched name", + # "name": "patched name", "notes": "these are some patched notes", "point": "POINT Z (-106.904107 34.068198 1408.3)", "release_status": "draft", @@ -91,7 +91,7 @@ def test_update_location(location): assert response.status_code == 200 data = response.json() assert data["id"] == location.id - assert data["name"] == payload["name"] + # assert data["name"] == payload["name"] assert data["notes"] == payload["notes"] assert data["point"] == payload["point"] assert data["release_status"] == payload["release_status"] @@ -115,9 +115,9 @@ def test_patch_location_404_not_found(location): Testing updating a location that does not exist """ bad_location_id = 99999 - location_name_patch = "another test name" + location_notes_patch = "patched notes" response = client.patch( - f"/location/{bad_location_id}", json={"name": location_name_patch} + f"/location/{bad_location_id}", json={"notes": location_notes_patch} ) data = response.json() assert response.status_code == 404 @@ -134,12 +134,15 @@ def test_get_locations(location): response = client.get("/location") assert response.status_code == 200 data = response.json() + from pprint import pprint + + pprint(data, indent=2) assert data["total"] == 1 assert data["items"][0]["id"] == location.id assert data["items"][0]["created_at"] == location.created_at.isoformat().replace( "+00:00", "Z" ) - assert data["items"][0]["name"] == location.name + # assert data["items"][0]["name"] == location.name assert data["items"][0]["notes"] == location.notes assert data["items"][0]["point"] == to_shape(location.point).wkt assert data["items"][0]["release_status"] == location.release_status @@ -158,7 +161,7 @@ def test_get_location_by_id(location): data = response.json() assert data["id"] == location.id assert data["created_at"] == location.created_at.isoformat().replace("+00:00", "Z") - assert data["name"] == location.name + # assert data["name"] == location.name assert data["point"] == to_shape(location.point).wkt assert data["release_status"] == location.release_status assert data["elevation_accuracy"] == location.elevation_accuracy From 97ea48fb51e40c2bb9d041e87e031a013a3b501d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 13:21:57 -0600 Subject: [PATCH 17/33] feat: skip record if error, not stop process --- transfers/thing_transfer.py | 6 +++++- transfers/well_transfer.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 3b48bfd6b..2d48cc679 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -39,7 +39,11 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - ) session.commit() - location = make_location(row) + try: + location = make_location(row) + except Exception as e: + logger.error(f"Error creating location for {row.PointID}: {e}") + continue session.add(location) payload = make_payload(row) thing_type = payload.pop("thing_type") diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 4b6103062..26cf58b1c 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -67,8 +67,8 @@ def transfer_wells(session, start_index=0, limit=0): try: location = make_location(row) except Exception as e: - logger.warning(f"Error making location for row {i}: {e}") - break + logger.warning(f"Error making location for {row.PointID}: {e}") + continue # print(location_row) session.add(location) From 22394a025201420b2f79a681c58aeb3ed5460830 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 13:22:20 -0600 Subject: [PATCH 18/33] refactor: increase timeout for location geographic requests --- services/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/util.py b/services/util.py index f134ecad1..6d2d5ace5 100644 --- a/services/util.py +++ b/services/util.py @@ -38,7 +38,7 @@ def get_tiger_data( "outFields": outfields, "returnGeometry": "false", } - resp = httpx.get(url, params=params, timeout=15) + resp = httpx.get(url, params=params, timeout=30) data = resp.json() if not data.get("features"): return None @@ -73,7 +73,7 @@ def get_quad_name_from_point(lon: float, lat: float) -> str: "returnGeometry": "false", } - resp = httpx.get(url, params=params, timeout=15) + resp = httpx.get(url, params=params, timeout=30) data = resp.json() if data["features"]: From 8fccdf43158f99e872fe13a24f5e0e88a178d1eb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 13:23:07 -0600 Subject: [PATCH 19/33] refactor: decrease limit for development --- transfers/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 7f387d4e4..5d5be3cbc 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -97,7 +97,7 @@ def main_transfer(): cleanup_wells_flag = True - limit = 100 + limit = 15 with session_ctx() as sess: if init: erase_and_initalize(sess) From 16ab98a980ff56a2262bb95e66749101d513634d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 15:40:29 -0600 Subject: [PATCH 20/33] fix: skip and log records with duplicates --- transfers/thing_transfer.py | 8 +++++++- transfers/well_transfer.py | 8 ++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 2d48cc679..39a688cf7 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -18,7 +18,7 @@ from db import LocationThingAssociation from services.thing_helper import add_thing -from transfers.util import make_location, read_csv, logger +from transfers.util import make_location, read_csv, logger, replace_nans def transfer_thing(session: Session, site_type: str, make_payload, limit=None) -> None: @@ -26,9 +26,15 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - ldf = read_csv("Location") ldf = ldf[ldf["SiteType"] == site_type] ldf = ldf[ldf["Easting"].notna() & ldf["Northing"].notna()] + ldf = replace_nans(ldf) n = len(ldf) start_time = time.time() for i, row in enumerate(ldf.itertuples()): + pointid = row.PointID + if ldf[ldf["PointID"] == pointid].shape[0] > 1: + logger.warning(f"PointID {pointid} has duplicate records. Skipping.") + continue + if limit and i >= limit: logger.warning(f"Reached limit of {limit} rows. Stopping migration.") break diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 26cf58b1c..bd70cb00c 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -36,14 +36,13 @@ ADDED = [] -def transfer_wells(session, start_index=0, limit=0): +def transfer_wells(session, limit=0): wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) ldf = read_csv("Location") ldf = ldf.drop(["PointID", "SSMA_TimeStamp"], axis=1) wdf = wdf.join(ldf.set_index("LocationId"), on="LocationId") wdf = wdf[wdf["SiteType"] == "GW"] wdf = wdf[wdf["Easting"].notna() & wdf["Northing"].notna()] - wdf = wdf.iloc[start_index : start_index + limit] wdf = replace_nans(wdf) @@ -54,6 +53,11 @@ def transfer_wells(session, start_index=0, limit=0): } made_things = [] for i, row in enumerate(wdf.itertuples()): + pointid = row.PointID + if wdf[wdf["PointID"] == pointid].shape[0] > 1: + logger.warning(f"PointID {pointid} has duplicate records. Skipping.") + continue + if limit and i >= limit: logger.warning("Reached limit of %d rows. Stopping migration.", limit) break From bf58bb6d265644f69182d779aa081ff9ee4128f9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 15:48:57 -0600 Subject: [PATCH 21/33] refactor: use pointid not index in log statement --- transfers/link_ids_transfer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transfers/link_ids_transfer.py b/transfers/link_ids_transfer.py index 987fac17d..4748a7e46 100644 --- a/transfers/link_ids_transfer.py +++ b/transfers/link_ids_transfer.py @@ -15,7 +15,6 @@ # =============================================================================== import re -import numpy as np import pandas as pd from db import Thing, ThingIdLink @@ -37,7 +36,9 @@ def transfer_link_ids_welldata(session): # RULE: exclude rows where both ids are null if pd.isna(row.OSEWellID) and pd.isna(row.OSEWelltagID): - logger.warning(f"Both OSEWellID and OSEWelltagID are null for row {i}") + logger.warning( + f"Both OSEWellID and OSEWelltagID are null for {row.PointID}" + ) continue thing = session.query(Thing).where(Thing.name == row.PointID).first() From 4040a54ccb5a32baad9c47c82cbc6f77e42ff9a9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 12 Sep 2025 17:02:44 -0600 Subject: [PATCH 22/33] WIP: work on location and dt transfers --- transfers/util.py | 81 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index e558e3e34..ef816d911 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from datetime import datetime +from datetime import datetime, timezone, timedelta +import pytz import re import io import logging @@ -124,11 +125,34 @@ def convert_to_wgs84_vertical_datum(row, z): return z -def make_location(row: pd.Series) -> Location: +def convert_mt_to_utc(dt_record: datetime): + """ + Developer's notes + + Assumes that records with time 00:00 (midnight) are meant to indicate a + date and therefore their timezone should just be set to UTC without + making any transformations + """ + t = dt_record.time() + if t.hour == 0 and t.minute == 0: + print("MIDNIGHT") + dt_record = dt_record.replace(tzinfo=timezone.utc) + else: + tz = pytz.timezone("America/Denver") + dt_record = tz.localize(dt_record) + if dt_record.dst() == timedelta(0): + print("MST", dt_record) + utc_offset = 7 + else: + print("MDT", dt_record) + utc_offset = 6 + dt_record = dt_record - timedelta(hours=utc_offset) + dt_record = dt_record.replace(tzinfo=timezone.utc) + print(dt_record) + return dt_record - # TODO: should the altitude be fetched from USGS' - # Elevation Point Query Service https://epqs.nationalmap.gov/v1/docs +def make_location(row: pd.Series) -> Location: point = Point(row.Easting, row.Northing) # Convert the point to a WGS84 coordinate system @@ -151,22 +175,55 @@ def make_location(row: pd.Series) -> Location: point_with_z = Point(point.x, point.y, z) - # TODO: determine correct created_at value - # created_at = row.DateCreated - name = row.PointID + if not (pd.isna(row.AltitudeMethod)): + elevation_method = lu_to_lexicon_map[f"LU_AltitudeMethod:{row.AltitudeMethod}"] + else: + elevation_method = None + + if not (pd.isna(row.CoordinateMethod)): + coordinate_method = lu_to_lexicon_map[ + f"LU_CoordinateMethod:{row.CoordinateMethod}" + ] + else: + coordinate_method = None + + """ + Developer's notes + + AMP folks said that the earlier date between DateCreated and SiteDate is when + the site was inventoried, whereas the later is when the record was made in + the database. This was because they were used interchangeably. + """ + if row.DateCreated and row.SiteDate: + + date_created = datetime.strptime(row.DateCreated, "%Y-%m-%d %H:%M:%S.%f") + site_date = datetime.strptime(row.SiteDate, "%Y-%m-%d %H:%M:%S.%f") + + if date_created > site_date: + created_at = date_created + else: + created_at = site_date + elif row.DateCreated and not row.SiteDate: + created_at = datetime.strptime(row.DateCreated, "%Y-%m-%d %H:%M:%S.%f") + else: + # TODO: should this be set to SiteDate if DateCreated is None and SiteDate is populated? + created_at = None + + # convert created_at from MST/MDT to UTC + if created_at is not None: + created_at = convert_mt_to_utc(created_at) location = Location( nma_pk_location=row.LocationId, - # TODO: determine if PointID should map to location.name or thing.name or if the Location table needs a name field at all. - name=row.PointID, + # name=row.PointID, point=point_with_z.wkt, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, - elevation_method=lu_to_lexicon_map[row.AltitudeMethod], - # created_at=created_at, + elevation_method=elevation_method, + created_at=created_at, # TODO: row.CoordinateAccuracy is not a float # coordinate_accuracy=row.CoordinateAccuracy, - coordinate_method=lu_to_lexicon_map[row.CoordinateMethod], + coordinate_method=coordinate_method, nma_coordinate_notes=row.CoordinateNotes, nma_notes_location=row.LocationNotes, state=state, From 6fef24901bad55afe2cfa6005fad4b915ce84c12 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 11:02:21 -0600 Subject: [PATCH 23/33] feat: convert times from mst/mdt to utc --- transfers/util.py | 11 +++++++---- transfers/waterlevels_transfer.py | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index ef816d911..873e03e0c 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -65,6 +65,9 @@ def flush(self): ) logger = logging.getLogger(__name__) +# workaround to not redirect httpx logging +logging.getLogger("httpx").setLevel(logging.WARNING) + # redirect stderr to the logger sys.stderr = StreamToLogger(logger, logging.ERROR) @@ -135,20 +138,20 @@ def convert_mt_to_utc(dt_record: datetime): """ t = dt_record.time() if t.hour == 0 and t.minute == 0: - print("MIDNIGHT") + # no time was measured, so just set the timezone to UTC and keep + # time at 00:00 dt_record = dt_record.replace(tzinfo=timezone.utc) else: tz = pytz.timezone("America/Denver") dt_record = tz.localize(dt_record) if dt_record.dst() == timedelta(0): - print("MST", dt_record) + # MST utc_offset = 7 else: - print("MDT", dt_record) + # MDT utc_offset = 6 dt_record = dt_record - timedelta(hours=utc_offset) dt_record = dt_record.replace(tzinfo=timezone.utc) - print(dt_record) return dt_record diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index 8e0546478..cfacb85da 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -20,7 +20,12 @@ import pandas as pd from db import Thing, Sample, Observation -from transfers.util import filter_to_valid_point_ids, logger, read_csv +from transfers.util import ( + filter_to_valid_point_ids, + logger, + read_csv, + convert_mt_to_utc, +) def transfer_water_levels(session): @@ -44,7 +49,14 @@ def transfer_water_levels(session): logger.warning(f"Skipping row {row.Index} due to missing data.") continue - dt = datetime.fromisoformat(row.DateMeasured) + if not pd.isna(row.TimeMeasured): + dt_measured = f"{row.DateMeasured} {row.TimeMeasured}" + else: + dt_measured = f"{row.DateMeasured} 12:00:00 AM" + + dt = datetime.strptime(dt_measured, "%Y-%m-%d %I:%M:%S %p") + dt_utc = convert_mt_to_utc(dt) + thing = session.query(Thing).where(Thing.name == row.PointID).first() if thing is None: logger.warning( @@ -57,7 +69,7 @@ def transfer_water_levels(session): sample.sample_type = "groundwater level" sample.field_sample_id = str(uuid.uuid4()) - sample.sample_date = dt + sample.sample_date = dt_utc sample.thing = thing session.add(sample) From c885ef52e1b0a9fc5e084a798bdc1f0ec93413a8 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 11:16:33 -0600 Subject: [PATCH 24/33] note: remove duplicative note --- transfers/util.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/transfers/util.py b/transfers/util.py index 873e03e0c..1f196c6dd 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -129,13 +129,6 @@ def convert_to_wgs84_vertical_datum(row, z): def convert_mt_to_utc(dt_record: datetime): - """ - Developer's notes - - Assumes that records with time 00:00 (midnight) are meant to indicate a - date and therefore their timezone should just be set to UTC without - making any transformations - """ t = dt_record.time() if t.hour == 0 and t.minute == 0: # no time was measured, so just set the timezone to UTC and keep @@ -169,8 +162,10 @@ def make_location(row: pd.Series) -> Location: z = row.Altitude if z: + elevation_from_epqs = False z = z * 0.3048 else: + elevation_from_epqs = True logger.info( f"Location {row.PointID} has no Altitude. Setting from National Map EPQS for " ) @@ -178,7 +173,9 @@ def make_location(row: pd.Series) -> Location: point_with_z = Point(point.x, point.y, z) - if not (pd.isna(row.AltitudeMethod)): + if elevation_from_epqs: + elevation_method = "USGS National Elevation Dataset (NED)" + elif not (pd.isna(row.AltitudeMethod)): elevation_method = lu_to_lexicon_map[f"LU_AltitudeMethod:{row.AltitudeMethod}"] else: elevation_method = None From ff98139a7f62da20514818dd6c80da9fd86fac0a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 12:44:47 -0600 Subject: [PATCH 25/33] refactor: wait for AMP feedback before transfering coord accuracy values --- transfers/util.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/transfers/util.py b/transfers/util.py index 1f196c6dd..b1c2c2f58 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -213,6 +213,69 @@ def make_location(row: pd.Series) -> Location: if created_at is not None: created_at = convert_mt_to_utc(created_at) + # TODO: AMP feedback is required for transfering coordinate accuracy values + # from NM_Aquifer to Ocotillo + # if row.CoordinateAccuracy == "U" or pd.isna(row.CoordinateAccuracy): + # # map "Unknown" to None + # row.CoordinateAccuracy = None + # elif row.CoordinateAccuracy == "5m": + # row.CoordinateAccuracy = 5.0 + # else: + # seconds = 0 + # minutes = 0 + # if row.CoordinateAccuracy == "1": + # seconds = 0.1 + # elif row.CoordinateAccuracy == "5": + # seconds = 0.5 + # elif row.CoordinateAccuracy == "F": + # seconds = 5 + # elif row.CoordinateAccuracy == "H": + # seconds = 0.01 + # elif row.CoordinateAccuracy == "M": + # minutes = 1 + # elif row.CoordinateAccuracy == "R": + # seconds = 3 + # elif row.CoordinateAccuracy == "S": + # seconds = 1 + # else: + # seconds = 10 + # coordinate_accuracy_decimal_deg = minutes/60 + seconds / 3600 + + # """ + # Developer's notes + + # To convert accuracy from decimal degrees to meters we do the following: + + # 1. Add the coordinate accuracy to both the latitude and longitude to + # find the "+" distance from the location + # 2. Convert "+" accuracy coordinates from decimal degrees to UTM Zone 13 + # N + # 3. Find the distance in meters from the original Easting/Northing and + # define this as the "+" accuracy in meters + # 4. Subtract the coordinate accuracy to both the latitude and longitude + # to find the "-" distance from the location + # 5. Convert the "-" accuracy coordinates from decimal degrees to UTM Zone + # 13 N + # 6. Find the distance in meters from the original Easting/Northing and + # define this as the "-" accuracy in meters + # 7. Set the coordinate accuracy in meters as the mean of the "+" and "-" + # distances from the location + # """ + # original_longitude = transformed_point.x + # original_latitude = transformed_point.y + + # plus_longitude = original_longitude + coordinate_accuracy_decimal_deg + # plus_latitude = original_latitude + coordinate_accuracy_decimal_deg + # plus_point_decimal_deg = Point(plus_longitude, plus_latitude) + # plus_point_utm_zone_13_n = transform_srid( + # plus_point_decimal_deg, + # SRID_WGS84, + # SRID_UTM_ZONE_13N) + + # minus_longitude = original_longitude - coordinate_accuracy_decimal_deg + # minus_latitude = original_latitude - coordinate_accuracy_decimal_deg + # minus_point_decimal_deg = Point(minus_longitude, minus_latitude) + location = Location( nma_pk_location=row.LocationId, # name=row.PointID, @@ -221,7 +284,7 @@ def make_location(row: pd.Series) -> Location: elevation_accuracy=row.AltitudeAccuracy, elevation_method=elevation_method, created_at=created_at, - # TODO: row.CoordinateAccuracy is not a float + # TODO: get AMP feedback on transfering these values. See above note # coordinate_accuracy=row.CoordinateAccuracy, coordinate_method=coordinate_method, nma_coordinate_notes=row.CoordinateNotes, From 1b8af65a59b758f1c634682a8a3dbdcdc08135e7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 13:52:46 -0600 Subject: [PATCH 26/33] refactor: made elevation separate from point field --- api/geospatial.py | 8 +++++--- db/location.py | 5 ++++- schemas/location.py | 5 +++++ services/geospatial_helper.py | 9 +++------ tests/conftest.py | 6 ++++-- tests/test_geospatial.py | 6 ++++-- tests/test_location.py | 10 ++++++++-- transfers/util.py | 5 ++--- transfers/waterlevels_transfer.py | 2 +- 9 files changed, 36 insertions(+), 20 deletions(-) diff --git a/api/geospatial.py b/api/geospatial.py index 4f25c2ff6..86951601d 100644 --- a/api/geospatial.py +++ b/api/geospatial.py @@ -14,7 +14,7 @@ # limitations under the License. # =============================================================================== import json -from typing import Annotated, List, Union +from typing import Annotated, List from fastapi import APIRouter, Query, HTTPException from fastapi.responses import FileResponse @@ -100,7 +100,9 @@ def get_feature_collection( things = get_thing_features(session, thing_type, group) - def make_feature_dict(thing, geometry, *other): + def make_feature_dict(thing, geometry, elevation, *other): + geometry = json.loads(geometry) + geometry["coordinates"].append(elevation) return { "type": "Feature", "properties": { @@ -109,7 +111,7 @@ def make_feature_dict(thing, geometry, *other): "name": thing.name, "group": group, }, - "geometry": json.loads(geometry), + "geometry": geometry, } features = [make_feature_dict(*item) for item in things] diff --git a/db/location.py b/db/location.py index bc16ad8eb..7cccd2db0 100644 --- a/db/location.py +++ b/db/location.py @@ -44,7 +44,10 @@ class Location(Base, AutoBaseMixin, ReleaseMixin): description: Mapped[str] = mapped_column # name: Mapped[str] = mapped_column(String(255), nullable=True) point: Mapped[WKBElement] = mapped_column( - Geometry(geometry_type="POINTZ", srid=SRID_WGS84, spatial_index=True) + Geometry(geometry_type="POINT", srid=SRID_WGS84, spatial_index=True) + ) + elevation: Mapped[float] = mapped_column( + nullable=False, comment="in meters with vertical datum of NAVD88" ) state: Mapped[str] = lexicon_term(nullable=True, default="New Mexico") diff --git a/schemas/location.py b/schemas/location.py index af26edc0a..8f5304588 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -37,6 +37,7 @@ class CreateLocation(BaseCreateModel): # name: str | None = None notes: str | None = None point: str # point is required and should be in WKT format + elevation: float release_status: str | None = "draft" elevation_accuracy: float | None = None elevation_method: str | None = None @@ -67,6 +68,9 @@ class LocationResponse(BaseResponseModel): # name: str | None notes: str | None point: str + elevation: float | None + horizontal_datum: str = "WGS84" + vertical_daum: str = "NAVD88" release_status: str | None elevation_accuracy: float | None elevation_method: str | None @@ -106,6 +110,7 @@ class UpdateLocation(BaseUpdateModel): # name: str | None = None notes: str | None = None point: str | None = None + elevation: float | None = None release_status: str | None = None elevation_accuracy: float | None = None elevation_method: str | None = None diff --git a/services/geospatial_helper.py b/services/geospatial_helper.py index 37189f4a4..0cebfe640 100644 --- a/services/geospatial_helper.py +++ b/services/geospatial_helper.py @@ -13,11 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import json - import shapefile from shapely.errors import GEOSException -from geoalchemy2 import functions as geofunc from shapely.io import from_geojson import constants @@ -46,7 +43,7 @@ def get_thing_features( # selection_args.append(SpringThing) sql = ( - select(Thing, ST_AsGeoJSON(Location.point).label("geojson")) + select(Thing, ST_AsGeoJSON(Location.point).label("geojson"), Location.elevation) .join(LocationThingAssociation, Thing.id == LocationThingAssociation.thing_id) .join(Location, LocationThingAssociation.location_id == Location.id) ) @@ -77,7 +74,7 @@ def create_shapefile(things: list, filename: str = "things.shp") -> None: shp.field("id", "L") shp.field("name", "C") - for thing, point in things: + for thing, point, elevation in things: # Assume loc.point is WKT or a Shapely geometry or GeoJSON if isinstance(point, str): try: @@ -88,7 +85,7 @@ def create_shapefile(things: list, filename: str = "things.shp") -> None: geom = to_shape(point) shp.point(geom.x, geom.y) - shp.record(thing.id, thing.name) + shp.record(thing.id, thing.name, elevation) def make_within_wkt(sql: Select, wkt: str) -> Select: diff --git a/tests/conftest.py b/tests/conftest.py index 3489334d0..2e14dbdc2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,8 @@ def location(): loc = Location( # name="first location", notes="these are some test notes", - point="POINT(-107.949533 33.809665 2464.9)", + point="POINT(-107.949533 33.809665)", + elevation=2464.9, release_status="draft", elevation_accuracy=100, elevation_method="Survey-grade GPS", @@ -34,7 +35,8 @@ def second_location(): with session_ctx() as session: location = Location( # name="second location", - point="POINT (10.2 10.2 0)", + point="POINT (10.2 10.2)", + elevation=0, release_status="draft", ) session.add(location) diff --git a/tests/test_geospatial.py b/tests/test_geospatial.py index 779e6a3c3..d8ff95e14 100644 --- a/tests/test_geospatial.py +++ b/tests/test_geospatial.py @@ -77,11 +77,13 @@ def populate(): loc1 = Location( # name="Test Location 1", - point=geofunc.ST_GeomFromText("POINT(10.1 10.1 0)", srid=SRID_WGS84), + point=geofunc.ST_GeomFromText("POINT(10.1 10.1)", srid=SRID_WGS84), + elevation=0, ) loc2 = Location( # name="Test Location 2", - point=geofunc.ST_GeomFromText("POINT(20 20 0)", srid=SRID_WGS84), + point=geofunc.ST_GeomFromText("POINT(20 20)", srid=SRID_WGS84), + elevation=0, ) session.add(loc1) session.add(loc2) diff --git a/tests/test_location.py b/tests/test_location.py index e2f849ac1..804db5632 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -44,7 +44,8 @@ def test_add_location(): payload = { # "name": "test location", "notes": "these are some test notes", - "point": "POINT Z (-106.607784 35.118924 1558.8)", + "point": "POINT (-106.607784 35.118924)", + "elevation": 1558.8, "release_status": "draft", "elevation_accuracy": 1.0, "elevation_method": "Survey-grade GPS", @@ -60,6 +61,7 @@ def test_add_location(): # assert data["name"] == payload["name"] assert data["notes"] == payload["notes"] assert data["point"] == payload["point"] + assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] assert data["elevation_accuracy"] == payload["elevation_accuracy"] assert data["elevation_method"] == payload["elevation_method"] @@ -80,7 +82,8 @@ def test_update_location(location): payload = { # "name": "patched name", "notes": "these are some patched notes", - "point": "POINT Z (-106.904107 34.068198 1408.3)", + "point": "POINT (-106.904107 34.068198)", + "elevation": 1408.3, "release_status": "draft", "elevation_accuracy": 2.0, "elevation_method": "Survey-grade GPS", @@ -94,6 +97,7 @@ def test_update_location(location): # assert data["name"] == payload["name"] assert data["notes"] == payload["notes"] assert data["point"] == payload["point"] + assert data["elevation"] == payload["elevation"] assert data["release_status"] == payload["release_status"] assert data["elevation_accuracy"] == payload["elevation_accuracy"] assert data["elevation_method"] == payload["elevation_method"] @@ -145,6 +149,7 @@ def test_get_locations(location): # assert data["items"][0]["name"] == location.name assert data["items"][0]["notes"] == location.notes assert data["items"][0]["point"] == to_shape(location.point).wkt + assert data["items"][0]["elevation"] == location.elevation assert data["items"][0]["release_status"] == location.release_status assert data["items"][0]["elevation_accuracy"] == location.elevation_accuracy assert data["items"][0]["elevation_method"] == location.elevation_method @@ -163,6 +168,7 @@ def test_get_location_by_id(location): assert data["created_at"] == location.created_at.isoformat().replace("+00:00", "Z") # assert data["name"] == location.name assert data["point"] == to_shape(location.point).wkt + assert data["elevation"] == location.elevation assert data["release_status"] == location.release_status assert data["elevation_accuracy"] == location.elevation_accuracy assert data["elevation_method"] == location.elevation_method diff --git a/transfers/util.py b/transfers/util.py index b1c2c2f58..327c55391 100644 --- a/transfers/util.py +++ b/transfers/util.py @@ -171,8 +171,6 @@ def make_location(row: pd.Series) -> Location: ) z = get_epqs_elevation_from_point(transformed_point.x, transformed_point.y) - point_with_z = Point(point.x, point.y, z) - if elevation_from_epqs: elevation_method = "USGS National Elevation Dataset (NED)" elif not (pd.isna(row.AltitudeMethod)): @@ -279,7 +277,8 @@ def make_location(row: pd.Series) -> Location: location = Location( nma_pk_location=row.LocationId, # name=row.PointID, - point=point_with_z.wkt, + point=point.wkt, + elevation=z, release_status="public" if row.PublicRelease else "private", elevation_accuracy=row.AltitudeAccuracy, elevation_method=elevation_method, diff --git a/transfers/waterlevels_transfer.py b/transfers/waterlevels_transfer.py index cfacb85da..53eff8d0c 100644 --- a/transfers/waterlevels_transfer.py +++ b/transfers/waterlevels_transfer.py @@ -36,7 +36,7 @@ def transfer_water_levels(session): start_time = time.time() for index, group in gwd: - logger.info(f"Processing PointID: {index}") + logger.info(f"Processing PointID: {index[0]}") n = len(group) for i, row in enumerate(group.itertuples()): if i and not i % 25: From dbfef6c6200d3ae277dc20178be3d613f593a628 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 14:12:52 -0600 Subject: [PATCH 27/33] fix: run tests for update branch names --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34a7c677a..12b7423d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ name: Tests on: pull_request: - branches: [ "main",'pre-production', 'transfer'] + branches: [ "production",'staging', 'transfer'] permissions: contents: read From 96dbff86964ffea2de285b92811243b612f5829a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 15 Sep 2025 14:22:19 -0600 Subject: [PATCH 28/33] fix: run tests on new branch names --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12b7423d5..8e7ef3d18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ name: Tests on: pull_request: - branches: [ "production",'staging', 'transfer'] + branches: ['production', 'staging', 'transfer'] permissions: contents: read From 4ecb5f1de34b68fc73931bcde891f474fa0a1b39 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 15 Sep 2025 20:45:16 -0600 Subject: [PATCH 29/33] feat: add documentation summarizing the purpose and contents of the db/base.py file and its mixins. feat: add StatusHistory model to log all time-variant operational statuses. --- db/base.py | 41 +++++++++++++++++++++++++++++++++++++++-- db/status_history.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 db/status_history.py diff --git a/db/base.py b/db/base.py index fe17e7905..6e72886e7 100644 --- a/db/base.py +++ b/db/base.py @@ -13,6 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +""" +db/base.py + +This file defines the foundational components for the SQLAlchemy models. +It includes: +1. The declarative base class (`Base`) that all models will inherit from. +2. A helper function (`lexicon_term`) to create standardized foreign key columns + referencing the `lexicon_term` table. +3. A helper function (`pascal_to_snake`) to convert class names from PascalCase to snake_case + for automatic table naming. +4. Mixins for common functionality: + - `AutoBaseMixin`: Adds automatic table naming and an auto-incrementing primary key. + - `PropertiesMixin`: Adds a JSONB properties column for storing additional attributes. + - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. + - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). +5. A simple `User` model for tracking user information in audit columns. +6. Polymorphic helper mixins (`HasStatusHistory`, `HasNotes`, `HasAttribution`, etc.) + which provide a clean, reusable way to add relationships to the polymorphic + metadata tables. Any model that can have a status history (like Thing or Location) + can simply inherit from the `HasStatusHistory` mixin. +7. An `AuditMixin` to add standard audit columns to tables. +""" + from sqlalchemy import ( Column, DateTime, @@ -20,8 +43,6 @@ Integer, JSON, String, - Boolean, - Text, ForeignKey, ) from sqlalchemy.orm import DeclarativeBase, declared_attr, Mapped, mapped_column @@ -41,6 +62,12 @@ class Base(DeclarativeBase): def lexicon_term(foreignkeykw=None, **kw): + """Create a SQLAlchemy mapped column for a self-referencing lexicon term. + + This helper function simplifies the creation of a string column that also + acts as a foreign key to the 'term' column of the 'lexicon_term' table. + It standardizes the column type to String(100) and sets the onupdate + behavior to "CASCADE".""" fkw = foreignkeykw if foreignkeykw else {} @@ -56,12 +83,16 @@ def pascal_to_snake(name): class ReleaseMixin: + """Mixin to add release status to a model.""" + @declared_attr def release_status(self): return lexicon_term(default="draft") class AuditMixin: + """Mixin to add standard audit columns to a model.""" + @declared_attr def created_at(self): return Column( @@ -109,6 +140,8 @@ def updated_by_id(self): class AutoBaseMixin(AuditMixin): + """Mixin to add automatic table naming and an auto-incrementing primary key.""" + @declared_attr def __tablename__(self): return pascal_to_snake(self.__name__) @@ -119,6 +152,8 @@ def id(self): class PropertiesMixin: + """Mixin to add a JSONB properties column for storing additional attributes.""" + @declared_attr def properties(self): return Column( @@ -130,6 +165,8 @@ def properties(self): class User(Base): + """Represents a user in the system.""" + __tablename__ = "user" id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False) diff --git a/db/status_history.py b/db/status_history.py new file mode 100644 index 000000000..0fb18ae08 --- /dev/null +++ b/db/status_history.py @@ -0,0 +1,38 @@ +""" +models/status_history.py + +This model defines the `StatusHistory` table, a central, polymorphic log for +all time-variant operational statuses (e.g., Use Status, Access Status). + +**NOTE**: This is a polymorphic table. It does not define outgoing relationships +itself. Instead, other tables (like Thing and Location) use the `HasStatusHistory` +mixin to establish a One-to-Many relationship TO this table. +""" + +import datetime + +from sqlalchemy import ( + Integer, + String, + DateTime, + Text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, ReleaseMixin + + +class StatusHistory(Base, AutoBaseMixin, ReleaseMixin): + status_type: Mapped[str] = mapped_column(String(50), nullable=False) + status_value: Mapped[str] = mapped_column(String(50), nullable=False) + start_date: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + end_date: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), nullable=True + ) + reason: Mapped[str] = mapped_column(Text, nullable=True) + + # Polymorphic relationship columns + statusable_id: Mapped[int] = mapped_column(Integer, nullable=False) + statusable_type: Mapped[str] = mapped_column(String(50), nullable=False) From b08461a4b6674ee6136a460abb3fe4abb7b07f27 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Mon, 15 Sep 2025 21:56:19 -0600 Subject: [PATCH 30/33] feat: add polymorphic helper mixin HasStatusHistory --- db/base.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/db/base.py b/db/base.py index 6e72886e7..f7c0e6e88 100644 --- a/db/base.py +++ b/db/base.py @@ -45,7 +45,13 @@ String, ForeignKey, ) -from sqlalchemy.orm import DeclarativeBase, declared_attr, Mapped, mapped_column +from sqlalchemy.orm import ( + DeclarativeBase, + declared_attr, + Mapped, + mapped_column, + relationship, +) from sqlalchemy_searchable import make_searchable from sqlalchemy_continuum import make_versioned import re @@ -82,6 +88,7 @@ def pascal_to_snake(name): return re.sub(r"(? Date: Wed, 17 Sep 2025 10:29:18 -0600 Subject: [PATCH 31/33] refactor: update mixin name from `HasStatusHistory` to `StatusHistoryMixin` --- db/base.py | 6 +++--- db/status_history.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/base.py b/db/base.py index f7c0e6e88..4da639a20 100644 --- a/db/base.py +++ b/db/base.py @@ -29,10 +29,10 @@ - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). 5. A simple `User` model for tracking user information in audit columns. -6. Polymorphic helper mixins (`HasStatusHistory`, `HasNotes`, `HasAttribution`, etc.) +6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `AttributionMixin`, etc.) which provide a clean, reusable way to add relationships to the polymorphic metadata tables. Any model that can have a status history (like Thing or Location) - can simply inherit from the `HasStatusHistory` mixin. + can simply inherit from the `StatusHistoryMixin` mixin. 7. An `AuditMixin` to add standard audit columns to tables. """ @@ -172,7 +172,7 @@ def properties(self): # ============= Polymorphic Helper Mixins ============================================= -class HasStatusHistory: +class StatusHistoryMixin: """ Mixin for models that can have a status history (e.g., Thing, Location). It automatically creates a polymorphic One-to-Many relationship to the diff --git a/db/status_history.py b/db/status_history.py index 0fb18ae08..acfd20f5d 100644 --- a/db/status_history.py +++ b/db/status_history.py @@ -5,7 +5,7 @@ all time-variant operational statuses (e.g., Use Status, Access Status). **NOTE**: This is a polymorphic table. It does not define outgoing relationships -itself. Instead, other tables (like Thing and Location) use the `HasStatusHistory` +itself. Instead, other tables (like Thing and Location) use the `StatusHistoryMixin` mixin to establish a One-to-Many relationship TO this table. """ From d6466f9500fae754c34b5e07d796f84039f0eebc Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Thu, 18 Sep 2025 13:48:22 -0600 Subject: [PATCH 32/33] update: create new `Permission` model --- db/permission.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 db/permission.py diff --git a/db/permission.py b/db/permission.py new file mode 100644 index 000000000..63c7e307d --- /dev/null +++ b/db/permission.py @@ -0,0 +1,85 @@ +""" +models/permission.py + +This model defines the `Permission` table, a polymorphic table that tracks +all legal and administrative agreements related to site access and activity. +Its purpose is to track who granted permission, what activities they authorized, +which entity the permission applies to, and for what period of time. +""" + +from typing import TYPE_CHECKING + +from sqlalchemy import ( + Integer, + ForeignKey, + String, + Boolean, + Date, + Text, + and_, +) +from sqlalchemy.orm import relationship, Mapped, mapped_column, foreign + +from db.base import Base, AutoBaseMixin, ReleaseMixin + +if TYPE_CHECKING: + from db.contact import Contact + from db.thing import Thing + from db.location import Location + + +class Permission(Base, AutoBaseMixin, ReleaseMixin): + """ + Represents a specific grant of permission from a Contact for a + specific entity (e.g., a Thing or Location). + """ + + # --- Foreign Keys --- + contact_id: Mapped[int] = mapped_column( + Integer, ForeignKey("contact.contact_id"), nullable=False + ) + + # --- Columns --- + allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + allow_installation: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False + ) + start_date: Mapped[Date] = mapped_column(Date, nullable=True) + end_date: Mapped[Date] = mapped_column(Date, nullable=True) + notes: Mapped[str] = mapped_column(Text, nullable=True) + + # --- Polymorphic Columns --- + permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) + permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) + + # --- Relationships --- + # Many-To-One: A Permission is granted by one Contact. + contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") + + # --- Polymorphic Parent Relationships (Internal) --- + # These are view-only relationships used by the 'target' property below. + # They tell SQLAlchemy exactly how to find the specific parent record for a given child. + _thing_target: Mapped["Thing"] = relationship( + "Thing", + primaryjoin=and_( + foreign(permissible_id) == Thing.thing_id, permissible_type == "Thing" + ), + viewonly=True, + ) + _location_target: Mapped["Location"] = relationship( + "Location", + primaryjoin=and_( + foreign(permissible_id) == Location.location_id, + permissible_type == "Location", + ), + viewonly=True, + ) + + @property + def target(self): + """ + A generic property to get the parent object (Thing, Location, etc.). + This is useful for simplifying application code by providing a single, + consistent way to access the parent of a polymorphic record. + """ + return getattr(self, f"_{self.permissible_type.lower()}_target") From 3eb3e41dccd680321233d25df834e9bbfb5dc7c1 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Thu, 18 Sep 2025 14:54:29 -0600 Subject: [PATCH 33/33] update: add `PermissionMixin` to `db/base.py` to automatically create a polymorphic One-to-Many relationship to the Permission table. --- db/base.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/db/base.py b/db/base.py index 4da639a20..31ec9ecd4 100644 --- a/db/base.py +++ b/db/base.py @@ -29,7 +29,7 @@ - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). 5. A simple `User` model for tracking user information in audit columns. -6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `AttributionMixin`, etc.) +6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `AttributionMixin`, `PermissionMixin`.) which provide a clean, reusable way to add relationships to the polymorphic metadata tables. Any model that can have a status history (like Thing or Location) can simply inherit from the `StatusHistoryMixin` mixin. @@ -191,6 +191,25 @@ def status_history(self): ) +class PermissionMixin: + """ + Mixin for models that can have permissions (e.g., Thing, Location). + It automatically creates a polymorphic One-to-Many relationship to the + Permission table. + """ + + @declared_attr + def permissions(self): + # One-to-Many polymorphic relationship + return relationship( + "Permission", + primaryjoin=f"and_({self.__name__}.{self.__name__.lower()}_id==Permission.permissible_id, " + f"Permission.permissible_type=='{self.__name__}')", + lazy="selectin", + viewonly=True, + ) + + class User(Base): """Represents a user in the system."""