From 81897d7dcbe71adbaddbc38d7eb85efe02238822 Mon Sep 17 00:00:00 2001 From: "Tadd.Bindas" Date: Sun, 31 Aug 2025 16:44:26 -0400 Subject: [PATCH 1/2] NGWPC 7009: update hydrovis with changes to work with t-route PR #30 --- Source/Ingest/docker/Dockerfile.ingest | 2 +- Source/RnR/dist/README.md | 87 ------------------ ...cefabric_manage-0.1.0-py2.py3-none-any.whl | Bin 2207 -> 0 bytes ...icefabric_tools-0.1.0-py2.py3-none-any.whl | Bin 4600 -> 0 bytes Source/RnR/docker/Dockerfile.process_flows | 30 ------ Source/RnR/docker/Dockerfile.troute | 31 ++----- Source/data/warehouse/.gitkeep | 0 Source/docker/README.md | 48 ++++++++-- Source/docker/compose.yaml | 15 --- Source/{docs => docker}/rnr_workflow.png | Bin Source/docker/run_post_process.sh | 4 +- Source/docker/start.sh | 2 +- Source/docs/workflow.md | 48 ---------- 13 files changed, 51 insertions(+), 216 deletions(-) delete mode 100644 Source/RnR/dist/README.md delete mode 100644 Source/RnR/dist/icefabric_manage-0.1.0-py2.py3-none-any.whl delete mode 100644 Source/RnR/dist/icefabric_tools-0.1.0-py2.py3-none-any.whl delete mode 100644 Source/RnR/docker/Dockerfile.process_flows delete mode 100644 Source/data/warehouse/.gitkeep rename Source/{docs => docker}/rnr_workflow.png (100%) delete mode 100644 Source/docs/workflow.md diff --git a/Source/Ingest/docker/Dockerfile.ingest b/Source/Ingest/docker/Dockerfile.ingest index 91b54bdfc..7cd632ba5 100644 --- a/Source/Ingest/docker/Dockerfile.ingest +++ b/Source/Ingest/docker/Dockerfile.ingest @@ -4,7 +4,7 @@ FROM python:3.12-slim RUN apt-get update && apt-get install -y curl # Install UV properly by copying from the official image -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/uv:0.8.14 /uv /uvx /bin/ # Set the working directory WORKDIR /app diff --git a/Source/RnR/dist/README.md b/Source/RnR/dist/README.md deleted file mode 100644 index 146466234..000000000 --- a/Source/RnR/dist/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Distributions - -The files in this repo are wheels from the internal NGWPC icefabric repo. Each package has a service(s) designed to extract RnR segments and format the hydrofabric v2.2 - -## How to use: -- These files are built in the T-Route RnR docker containers and run the `rnr.get_rnr_segment()` -- These Wheels are required to run `troute-rnr/` - -## Why are there wheels instead of a github install? -- Since the `icefabric/` repo has not been delivered at this time, we've attached wheels from a version that works with replace and route to ensure the code will run - -## The `get_rnr_segment()` Function used in RnR - -### Overview - -The `get_rnr_segment` function extracts a river network segment from a hydrofabric dataset based on RnR (Reach and Route) rules. It processes hydrologic data from an Iceberg catalog and exports the results as a multi-layer GeoPackage file. - -### Function Signature - -```python -def get_rnr_segment(catalog: Catalog, reach_id: str, output_file: str) -> gpd.GeoDataFrame -``` - -### Parameters - -| Parameter | Type | Description | -|-----------|------|-------------| -| `catalog` | `Catalog` | The Iceberg catalog containing the hydrofabric data tables | -| `reach_id` | `str` | The reach identifier (hf_id) from the NWPS API used to locate the target stream segment | -| `output_file` | `str` | File path where the output GeoPackage will be saved | - -### Returns - -- **Type**: `gpd.GeoDataFrame` -- **Description**: A GeoDataFrame containing the processed hydrofabric data - -### Data Sources - -The function accesses the following tables from the hydrofabric. These are parquet files of each layer which are available from the raytheon s3:// or can be built from the hydrofabric v2.2 gpkg file through a geopandas parquet conversion. - -### Processing Logic - -1. **Stream Identification**: Locates the target stream using the provided `reach_id` -2. **Mainstem Filtering**: Identifies all streams on the same mainstem with hydrosequence values less than or equal to the target stream -3. **Stream Order Matching**: Filters features to include only those with the same stream order as the target reach -4. **Geometric Data Extraction**: Retrieves associated geometric and attribute data for: - - Flowpaths - - Nexus points - - Divides - - Points of interest - - Hydrolocations -5. **Multi-layer Export**: Saves all processed data as separate layers in a single GeoPackage file - -### Output Structure - -The function creates a GeoPackage file with the following layers: -- `flowpaths` -- `nexus` -- `divides` -- `divide-attributes` -- `network` -- `pois` -- `flowpath-attributes-ml` -- `flowpath-attributes` -- `hydrolocations` - -### Usage Example - -```python -from icefabric_manage import build -from icefabric_tools import rnr -from pyiceberg.catalog import load_catalog - -# Builds the catalog -data_dir = -self.catalog_settings = { - "type": "sql", - "uri": f"sqlite:///{str(data_dir / 'warehouse/pyiceberg_catalog.db')}", - "warehouse": f"file://{str(data_dir.resolve())}/warehouse", -} - -self.catalog = load_catalog("hydrofabric", **self.catalog_settings) -build(self.catalog, Path(f"{self.data_dir.resolve()}/parquet")) - -# Extract RnR segment -gdf = rnr.get_rnr_segment(settings.catalog, inputs.reach.id, settings.tmp_geopackage) -``` diff --git a/Source/RnR/dist/icefabric_manage-0.1.0-py2.py3-none-any.whl b/Source/RnR/dist/icefabric_manage-0.1.0-py2.py3-none-any.whl deleted file mode 100644 index e7a6c90255d6bd0d7be1340b5033f5a3309710a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2207 zcmWIWW@Zs#U|`??VyA$&{AaJ*09m#`ECa-u$*F0HNky5-@wth4iRr2O@$s2?nI-Y@ zdIgo<+NaKkp78PW_4d&^<$82o3 z2!6%iF~u`>i(mE1Rcqf1nqB8y#jH1>H1KNueb(fj%VoiTn;MHRSWd6j@7cIEtj{~O zv0U~;oKtmB@8nrwcN(oTb~H(4zOWKm;D64maY6Jg*9pAa_LyzDDZBGgb;~?+eiNf( zU5iYAd2qkGa8l~Y#_Pup%KCOI`Ta6Bce4+*HSbq1IiUNce`49P#n~pmS3T@2ud_G>Hi?QWz*rg;+y%T!X8xkYIn&O@?M+ybQi-fj~`nF zg5EYB);Jw}@MXe}0x>uKX$vKnRy>k5FPY+;eE5r2ZKBmIkETPb{;zxZ?oaZDS$tn! zd+PHv2|xVTptxf3qL@?$-(w0EvkeccwfNTW=H~W~u@q_Uod3el?W|18|LXHq!9^DO zlXh;IQGED4@5xOF6&w^>ux>)e20z!q_?Y=*d+d7uLz%+LRi-ZpychAxbRSyys8;=8itLX{G-r z;rF7Q6Vg0(s>akdMz1xPdMy0>=J%pZSMK~0TgbFvme-Fn{jw%eTPL0BYMIEgl4Ium zeQWIk+}EEzp?=xaR9@=Xl{;}^!jiu|3?uh@{AFFzwfj@A)!9jx+2^e;_MBV1xv^+s zYsSVXr`5a{riEx;weos6!C~4IW!2(oXT@L62>6`5+~gJSNu8T!2UyPpOt@jk%w)Cg z%)#eudnOg}1Z{m^61(Yqa(vq96QB5smVGy}&YU#gbm3W^;K+9O2LBJXZI%i1mRMi7 z?Bx1BAhN1v*{6(PjW*-thyNE#Y}W9bm>HRoG(q|0$jXrau4M@<5^Zp}WTv@Q5{;K}>R z%L80qwjTD{mX;=O?9HWNnDEW#oOU+XiJz9$4!*1!j;ibC3gt`jh8_7fM^OH=VD+ml zZr8;P@})Gam7Q2N1}<#BWF^pNpbH4Yl;!>QGq`d3{)GnE*iB36}#bNrFhgI-; zR?g99u`AY|_E?<76|rzi!Rj6t{_w)`lh+SFZ;KJQwlyZA_UyD3J{_+Auh?E%7Ti~} zEx=ABU|Dn8W<&Yo-TVi;a=KT#D#;x5U4Gjx@n`CmFARUep4s&K$u-$prEd?kw!OCF z?PAaDqO0BaO(JzfUzD0je|&jAtS)Go=Hfq1{uSrMp0oz%ZnD|ES#1L6%~koyHyZ5@ zMfQA-y|p&+4a>XljymNpH(Od{>E>jsW(kxT+&H{$p}bOj`KpPUx9)rLJ#>>}3*79F z8Zm9h!!n~l5dy?(QEhpVd(_jwQBvln^2b+yi&Ilno`;EM5sBLB1Ax?Vab^*05b z@X+x*cjDw3Z+0(KoKYb1= z75=Q1sC0}&5Uup&D|`E$0hnxhfmodw*9Eyc`vxac(m&yC$t|sXR{fO3Ityo;x`?0dUcE2fOP}-Z;Gu7Yvts{PJh+<#S zx6M|h;CHL3oJ5eQ@uV&Ra}y=&``;${ma%p*_KF8IZBlZcX=Ld6CaCmt@8lWVX3jXz zxU%e0<(t5ck81=uz0KWT#+|)zGN)92=fmZElMiy3tz*8xw3mOwbBl+;^P}QcH`>Rn zeE8BP=Fse&TE}OU-@kl!)*ZQb8#8uI({kh#S+Vze^6^{8%osM?2Y53wi7?}?`hXz` z21^=26uycO-6Ztt3}Oxg!;;1_tR_J!H*{0bO9QwmjJ3>|?!j3$pc{*xmJ!B^u@Ye{ jIAx=oiJk%xW^N(IOh|eR@MdKLX=DY$3BaWM6U+kuCI)ZS diff --git a/Source/RnR/dist/icefabric_tools-0.1.0-py2.py3-none-any.whl b/Source/RnR/dist/icefabric_tools-0.1.0-py2.py3-none-any.whl deleted file mode 100644 index b62a8caecf1309fe58e01b03967aaf833ddacc39..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4600 zcma)=cQjn<*T;v6Zld=RM52tI=)IRQ%IMwb!XSuVM@ftzB+;YI1aY+>qDL@#jR;Yr zOLT$}39tLRYrS_}_x|4Zp0oBk=lt_o-{(BLJbN2JfY)dN008l&sTl005u$@H-{vlj z<?jBy1ZO zm}~66d{;GYAOh7N5SnIl9tQPq~;u-u!u_fJX zat0v)pnkashJR8F_i?{e+iGs*JtILIF;d0x@!?b)7(>ws0P;V1qs?HyiQ=9SxfZPq zc?ON8Hc6{-^#8Rh`w+ux*e0MF&7{R`B)8BQDd(HUz3YtNtr9O(p`7-C)j#lRh^4D7 zj$f&}4IyOUG*@==xGT(*#JEVQbofY;gQZnx>#V*#4f@C= zNGx5{pU?d5*+GV)@I{l=?8143_{|es7h=+5TIBU+Jt9Bd)zy^%{6c)wrVx|(zRRk9 zbdb@u>U%D&oFv6`DN}T0iqXNE&HCWa$L&P;518=v9&@9{kbBdFfdHsXNySIkGB+%_ z(R4&Ojl%Kf6sed*=T66nJTFPJ%qoxgg4lSO49sJ<)rjgKiOd(UcRu}Fjg+ARPoI*9 z*v%@t6t=9U>KTl3tDuTARG`=FnYB{E+J@9==sPYg`qeoigF6P69$q*? z$_$QcHTh0M_Ghj4aBQ-lsmqiyrj;91mF;OBcW5^SS!yPD zR)@VJp31;6ieh9aeIa0i@x~=cG2;Du$;v2OrMSG*)$8;+)1<0ltzCSahI-dG-ti%_ zwS?5ICNM>}?aD2-7NC|JY)Jiq@OEso^qxK=S6uo6z(Ym{Q3blVI$ zj`r5~ERZKkPZ$55U8hB)ELqC3%)nCBuX^QZr8oT=Wo(!h_Q4h-Z z+emtN2)JLGiLl@A`G-n|J89P8ZD&#!i%UI;D=)W@+JxdtY?Rj94%4E#r#88F>W^jJ zO8oX|N4B0HpUudsA<<~?nu%fepcwJwg<+l^(P}oxWNJpS)a~ppOnd)}5uM$;U zEbD|4u}1+NQA*ZKaCRay;M&ujV28N&q9WZnQ~6yMd^(UoCijK~dWEpNWyv{v-09ma z*(TFQ$Ob4#*M0oaoZ9F;3xfl>9zK5TxSf7piQCp4-#hoJUAbxOY39`{#46WGTR-iU zDVe`98&iJKDT%vF&3D>PyFO7OWO?GTCruvdu_3mYq8h4&m?Up1>LB&FMsFo#Qph&Z zv?!aoJ$2gs1V##f#mc_0)WvU3kmDw4d+)Zn7e@?p8NJfYm<(EoUXzMSENZgl9(Wz; zV%JMJz#XoAp&6|p75X!54soFF@^IU6yrkmmm#Q2^u7>)x(b!J;Ua%EuMD&6=kbXv1 z0FvT%sGGb`hTo&{BLV;QzE4-%g2jNN*FK$Jwt4aaN|L%4L!sl;9m)N)6iq~RAXqX( zxZr7iU-$g!mM%ZMK-D8GPW80EUV;C_!>n1)GNmi^d+2F$qiw2(q+@sV!;Nn7g)!@l zP60CbjcO~!8W)Ui+mE3IpOe!`_N|#lqv0-M(F{7_WMCyEmkPd2xk>zykCo_vE0H50 z(a2!u^uE-P-jm3z+3%}kn&7guzI2gzeZ4@J z5Dn^^a}UPr=2UL(Pmj4T&3Ao`_k}OG%K{oK#e-HKT%&+9R>W)hyPhrg0xifUWkV~Y zc=3#wVRcDgq>Nw(#tYz`~0PoYFQj3z2tAtT3{bbvR? z(f6%cOxI#3Xc_ie7Z)O<^7=eh%Tn8~`ve3pSr=8fJh+g&?==YXHCz z1pq+*54FtK%lERBRhU7%mnA6M&dexhRh^VXLnOZ7E02Sa^f?@$(HJ;!o!VzlUKj+n zPl!D~xqc&!Z*0B+LC%w|ac80B;mybUpfaYCUah)}7L`~(jN5>=HWd{etmY1_v;LBM za)9x-bVd!S4=)`m#u%4BKNhbr?p8o3Bi*ho*jOF;kBM*RyiBO)VLOegFs)JEW&wXq zFPkgkj`wL)jXq+ClwB=auvCBJ0^7(t)G(I*apa+is~sF`?H$`~-*4erQkFg#i~*=Y z>rv0+Qqw4w;ytAaV4}EGG|o#R$E1!>7+M(1e|JvYI8z|Ac?&ee8G@N*D7WN77$=`AFHs(9mwwKwmwd^aNZgt#yPn4Hu!ePAl`lrY-VvCAKH5>s2~bkt+#5A+ zTsr^=*1uEp{lc~>@JRHV19mb!2T|Ck*b#ev4r2*9KQxuX6fItWwwf04^hdqtZm~YrPa{)y=}`JU=U`+Tklw8Z~&kztfp_`<^rZ|(`{^;B-uFP^(=%p zhxZ&33<@Y#i+RITGe_eku=kyiTA5F?up81o?0mL%>iU%uiDn(LWs`~sE=}=YG8!+) zXTR`QLkoe@SV8AnXQb(S?e^H$c`x=j6`fUx7kIvBkwiXwuT+Q`+h`x0A0h>X-zc?* zshd0rif9N|dK+LbQE=n_0sb7BstZ}JFt2W?e6kyqgq(SDbv^CwJ`Yr1Noc8(rIMy0 zG!?$^_6dUco z6ifr;ehq&k5lAR*Xwv6<+<#X0>3kzWzccV`!v?DTj+?r5e(aYv)>Zr*Y;vxfo2Fn2 zmjY~$Xn!$42T#)~sz9&zi7o=gu`d+M-am6xKx9Ph%1VBI*p3P?)PWO~N4@h5Z-c6U z^GDR}qjDd=I7k}qV<|s8Zn<%rPeI6jfTpx3HSw2it(-h1S6c*|&F#5d5iG>2V~5li zvryGDz}-#tbOw{Y{A0k#m4|GeWaKOW_j=Ew4?V>bJI7C1%g!-WHr0iy+wAQB$h$BG8io<%%U_lDsi=nBArLgke^?2*mQ z*&|oeTK6*gt_L0f@B$wI;Qf1A6BH2=6%rA0a`*EWboX-c7KW-DE2}9RD?f)p^;t#9 zTi4CZm!?T-YhO~M^(GLFK2bn*Yf6=@u3EK~#4e-E$P}P+D!q`r8fx`q%k}I9Cm&i% z+$=LOsqmp*1l@YSBZcWQrrYyK5W9hzQnqYNtH9j}tvdM7*)5&9R*>+t-@?)!#pJJ{ zF5&sEJ$LY3J}Du&OgYt5yGRv3Suv~pC|ldfwV^WWS=5kjps3`=^9SCW!E_Jym>7{>6rwX{oF0l8tCVhsHnrIn~G8yTdY-$>(kM6_w)%5OmmgMy>js1wlY|nYg)B=dJ zr)7EG=gr2nKK`UqG?{y}KHSf20p4V4m%)}YS7i!uvixS4?)#u7=Y4QBHv`)Gx?gmC z-2^`f&*oz4*Pi2+)I1HtdX19N$tNeEXL=r~)qvJ-)lR7!o}!CBG?SVE!z=EjPPmJ( zjj0GJ{4bjn1`s@aTHt>l3S9p5|6G{p-yi?Sh~O&dYOnqW1^~cB*ZdRckIwxnPgki|!{$#a;r0KY`kx@WioELVe Date: Tue, 2 Sep 2025 17:08:46 -0400 Subject: [PATCH 2/2] changed t-route branch reference --- Source/RnR/docker/Dockerfile.troute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/RnR/docker/Dockerfile.troute b/Source/RnR/docker/Dockerfile.troute index 75a572ec7..181a5ce00 100644 --- a/Source/RnR/docker/Dockerfile.troute +++ b/Source/RnR/docker/Dockerfile.troute @@ -13,7 +13,7 @@ RUN uv python install 3.11 RUN git clone https://github.com/NGWPC/t-route.git WORKDIR "/t-route/" -RUN git checkout NGWPC-7009 +RUN git checkout pi-7-NGWPC-6258 # # Create netcdf symlink with error handling RUN ln -s /usr/lib64/gfortran/modules/netcdf.mod /usr/include/openmpi-x86_64/netcdf.mod || echo "NetCDF module link creation failed but continuing"