From c936d6184a70108912e54b98b588a1e4a5b50268 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 11:24:59 -0500 Subject: [PATCH 01/21] wip makefile, github workflow --- .github/workflows/test.yml | 75 +++++++++++++++++++++++ Makefile | 120 +++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 Makefile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b640d9e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,75 @@ +--- +name: ci + +'on': + push: + branches: + - '**' + +permissions: + contents: write + +jobs: + ci: + runs-on: [self-hosted, dev] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extras / Count Lines of Source Code + run: make extras/cloc + + - name: Install requirements + run: make init + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Build [Dev] (and Deploy) + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - name: Build (dev) + if: github.ref == 'refs/heads/dev' + run: pnpm build:dev + + - name: Deploy (dev) [Copy static files over] + if: github.ref == 'refs/heads/dev' + run: rm -rf /var/www/app/* && mv build/* /var/www/app/ + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Lint + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + - name: Lint + run: pnpm lint + + - name: Check + run: pnpm check + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Build [Prod] (and Upload binary) + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + - name: Check release version + id: check-release-version + if: github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ github.token }} + run: | + RELEASE_TAG=v$(jq -r .version package.json) + echo RELEASE_TAG=$RELEASE_TAG + test -n "$RELEASE_TAG" + # Test that github-cli is working + gh --version + gh release list -L 1 + # TODO: enhance this to be: if release_tag > current_prod_tag, deploy + gh release view $RELEASE_TAG || echo "PUBLISH=1" >> "$GITHUB_OUTPUT" + + # yamllint disable rule:line-length + - name: Build (production release) + if: github.ref == 'refs/heads/master' && steps.check-release-version.outputs.PUBLISH + env: + GH_TOKEN: ${{ github.token }} + run: set -o pipefail; make build + + - name: Upload artifacts (production release) + if: github.ref == 'refs/heads/master' && steps.check-release-version.outputs.PUBLISH + env: + GH_TOKEN: ${{ github.token }} + run: set -o pipefail; make deploy/upload diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9edc19a --- /dev/null +++ b/Makefile @@ -0,0 +1,120 @@ +SHELL=/bin/bash + +.DEFAULT_GOAL := _help + +# NOTE: must put a character and two pound "\t##" to show up in this list. Keep it brief! IGNORE_ME +.PHONY: _help +_help: + @grep -h "##" $(MAKEFILE_LIST) | grep -v IGNORE_ME | sed -e 's/##//' | column -t -s $$'\t' + + + +# --------------------------------------- +# Install requirements +# --------------------------------------- + +.PHONY: init +init: ## Install requirements (w/o --frozen-lockfile) + # Check version + [[ "$(shell pnpm --version)" =~ "8." ]] + # Remove old install + rm -rf node_modules/ + # Install requirements + pnpm install + # Sync svelte kit (optional) + pnpm svelte-kit sync + + + +# --------------------------------------- +# Run, lint & format +# --------------------------------------- + +.PHONY: run +run: + pnpm dev + +.PHONY: format +format: ## pnpm format + pnpm format + +.PHONY: lint +lint: ## pnpm lint && pnpm check + pnpm lint + pnpm check + + + +# --------------------------------------- +# Build & install +# --------------------------------------- + +APP_VERSION ?= v$(shell jq -r .version package.json) +APP_BUNDLE ?= build-${APP_VERSION}.tar.xz +APP_RELEASE_DATE ?= $(shell date --iso) + +.PHONY: build +build: clean +build: ## Build the release + ./sql/build.sh + du -h ${APP_BUNDLE} + +.PHONY: deploy/upload +deploy/upload: ## Upload to GitHub releases + test -n "${APP_VERSION}" + test -f ${APP_BUNDLE} + gh release create ${APP_VERSION} --generate-notes + gh release upload ${APP_VERSION} ${APP_BUNDLE} + +.PHONY: deploy/delete +deploy/delete: + gh release delete ${APP_VERSION} + git push origin --delete ${APP_VERSION} + - git tag -d ${APP_VERSION} + + +REMOTE_HEAD ?= origin/master + +.PHONY: _check-git-up-to-date +_check-git-up-to-date: + git branch --show-current + git fetch + # Check that we are in sync with ${REMOTE_HEAD} + git diff --quiet ${REMOTE_HEAD} + +PROJECT_NAME ?= web +DEPLOY_URL ?= https://nutra.tk/ + +.PHONY: deploy/install-prod +deploy/install-prod: _check-git-up-to-date +deploy/install-prod: ## Install (on prod VPS) + # Check the version string was extracted from package.json + test -n "${APP_VERSION}" + # Download ${APP_VERSION} + curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${APP_VERSION}/${APP_BUNDLE} + tar xf ${APP_BUNDLE} + rm -f ${APP_BUNDLE} + # Copy in place + rm -rf /var/www/app/* && mv build/* /var/www/app/ + # Test live URL + curl -fI ${DEPLOY_URL} + + + +# --------------------------------------- +# Clean & extras +# --------------------------------------- + +CLEAN_LOCS_ROOT ?= *.tar.xz build/ + +.PHONY: clean +clean: ## Clean up leftover bits and stuff from build + rm -rf ${CLEAN_LOCS_ROOT} + +.PHONY: purge +purge: ## Purge package-lock.json && node_modules/ + rm -rf package-lock.json pnpm-lock.yaml node_modules/ + +.PHONY: extras/cloc +extras/cloc: + cloc HEAD --exclude-dir=svelte.config.js,pnpm-lock.yaml,package-lock.json From 10b209e7431d508074c2c88434e7b55227e690ea Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 12:22:30 -0500 Subject: [PATCH 02/21] wip format --- Makefile | 4 ++-- sql/format-sql.sh | 6 ------ sql/format.sh | 7 +++++++ sql/tables.sql | 16 +++++++++------- 4 files changed, 18 insertions(+), 15 deletions(-) delete mode 100755 sql/format-sql.sh create mode 100755 sql/format.sh diff --git a/Makefile b/Makefile index 9edc19a..c029d44 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,8 @@ run: pnpm dev .PHONY: format -format: ## pnpm format - pnpm format +format: ## format SQL with pg_format + ./sql/format.sh .PHONY: lint lint: ## pnpm lint && pnpm check diff --git a/sql/format-sql.sh b/sql/format-sql.sh deleted file mode 100755 index 32b0965..0000000 --- a/sql/format-sql.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")" - -pg_format -s 2 tables.sql -o tables.sql -# pg_format -s 2 import.sql -o import.sql diff --git a/sql/format.sh b/sql/format.sh new file mode 100755 index 0000000..8cc5c17 --- /dev/null +++ b/sql/format.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +# TODO: what about import.sql? It gets formatted too ugly +pg_format -L -s 2 -w 100 tables.sql >tables.fmt.sql +mv tables.fmt.sql tables.sql diff --git a/sql/tables.sql b/sql/tables.sql index 9196e3c..3b135ae 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -13,8 +13,11 @@ -- -- You should have received a copy of the GNU General Public License -- along with this program. If not, see . - -CREATE TABLE version( id integer PRIMARY KEY AUTOINCREMENT, version text NOT NULL, created timestamp DEFAULT CURRENT_TIMESTAMP, notes text +CREATE TABLE version ( + id integer PRIMARY KEY AUTOINCREMENT, + version text NOT NULL, + created timestamp DEFAULT CURRENT_TIMESTAMP, + notes text ); CREATE TABLE nutr_def ( @@ -44,10 +47,10 @@ CREATE TABLE food_des ( ref_desc text, refuse int, sci_name text, - n_factor FLOAT, - pro_factor FLOAT, - fat_factor FLOAT, - cho_factor FLOAT, + n_factor float, + pro_factor float, + fat_factor float, + cho_factor float, FOREIGN KEY (fdgrp_id) REFERENCES fdgrp (id) ); @@ -143,4 +146,3 @@ CREATE TABLE serving ( FOREIGN KEY (food_id) REFERENCES food_des (id), FOREIGN KEY (msre_id) REFERENCES serv_desc (id) ); - From fa3c02f95e3513ec3cdc9481921a7cd8805ef8e3 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 12:45:44 -0500 Subject: [PATCH 03/21] fixup makefile more --- Makefile | 65 +++++++++++++++---------------------------- sql/latest_version.py | 23 +++++++++++++++ 2 files changed, 46 insertions(+), 42 deletions(-) create mode 100644 sql/latest_version.py diff --git a/Makefile b/Makefile index c029d44..d9afdff 100644 --- a/Makefile +++ b/Makefile @@ -27,21 +27,14 @@ init: ## Install requirements (w/o --frozen-lockfile) # --------------------------------------- -# Run, lint & format +# Format # --------------------------------------- -.PHONY: run -run: - pnpm dev - .PHONY: format format: ## format SQL with pg_format - ./sql/format.sh - -.PHONY: lint -lint: ## pnpm lint && pnpm check - pnpm lint - pnpm check + # TODO: what about import.sql? It gets formatted too ugly + pg_format -L -s 2 -w 100 sql/tables.sql >sql/tables.fmt.sql + mv sql/tables.fmt.sql sql/tables.sql\ @@ -49,28 +42,27 @@ lint: ## pnpm lint && pnpm check # Build & install # --------------------------------------- -APP_VERSION ?= v$(shell jq -r .version package.json) -APP_BUNDLE ?= build-${APP_VERSION}.tar.xz -APP_RELEASE_DATE ?= $(shell date --iso) +DB_VERSION ?= $(shell python3 sql/latest_version.py) +DB_FILE ?= sql/dist/usda.sqlite3-${DB_VERSION}.tar.xz .PHONY: build build: clean build: ## Build the release - ./sql/build.sh - du -h ${APP_BUNDLE} + ./sql/build.sh ${DB_VERSION} + du -h ${DB_FILE} .PHONY: deploy/upload deploy/upload: ## Upload to GitHub releases - test -n "${APP_VERSION}" - test -f ${APP_BUNDLE} - gh release create ${APP_VERSION} --generate-notes - gh release upload ${APP_VERSION} ${APP_BUNDLE} + test -n "${DB_VERSION}" + test -f ${DB_FILE} + gh release create v${DB_VERSION} --generate-notes + gh release upload v${DB_VERSION} ${DB_FILE} .PHONY: deploy/delete deploy/delete: - gh release delete ${APP_VERSION} - git push origin --delete ${APP_VERSION} - - git tag -d ${APP_VERSION} + gh release delete v${DB_VERSION} + git push origin --delete v${DB_VERSION} + - git tag -d v${DB_VERSION} REMOTE_HEAD ?= origin/master @@ -82,22 +74,16 @@ _check-git-up-to-date: # Check that we are in sync with ${REMOTE_HEAD} git diff --quiet ${REMOTE_HEAD} -PROJECT_NAME ?= web -DEPLOY_URL ?= https://nutra.tk/ +PROJECT_NAME ?= usda-sqlite .PHONY: deploy/install-prod deploy/install-prod: _check-git-up-to-date deploy/install-prod: ## Install (on prod VPS) # Check the version string was extracted from package.json - test -n "${APP_VERSION}" - # Download ${APP_VERSION} - curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${APP_VERSION}/${APP_BUNDLE} - tar xf ${APP_BUNDLE} - rm -f ${APP_BUNDLE} - # Copy in place - rm -rf /var/www/app/* && mv build/* /var/www/app/ - # Test live URL - curl -fI ${DEPLOY_URL} + test -n "${DB_VERSION}" + # Download ${DB_VERSION} + curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${DB_VERSION}/${DB_FILE} + tar xf ${DB_FILE} @@ -105,16 +91,11 @@ deploy/install-prod: ## Install (on prod VPS) # Clean & extras # --------------------------------------- -CLEAN_LOCS_ROOT ?= *.tar.xz build/ - .PHONY: clean clean: ## Clean up leftover bits and stuff from build - rm -rf ${CLEAN_LOCS_ROOT} - -.PHONY: purge -purge: ## Purge package-lock.json && node_modules/ - rm -rf package-lock.json pnpm-lock.yaml node_modules/ + rm -f sql/*.sqlite + rm -f sql/*.sqlite3 .PHONY: extras/cloc extras/cloc: - cloc HEAD --exclude-dir=svelte.config.js,pnpm-lock.yaml,package-lock.json + cloc HEAD --exclude-dir=usda.svg diff --git a/sql/latest_version.py b/sql/latest_version.py new file mode 100644 index 0000000..a5214ca --- /dev/null +++ b/sql/latest_version.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Sat Mar 2 12:32:45 2024 + +@author: shane +""" + +import csv +import os + +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) + +version_csv_path = os.path.join( + SCRIPT_DIR, "version.csv" +) + +rows = [] +with open(version_csv_path, "r", encoding="utf-8") as _r_file: + reader = csv.reader(_r_file) + rows = list(reader) + +print(rows[-1][1], end="") From 3aca76b83bdd9aed79b64a298e5144ac02580084 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 12:46:55 -0500 Subject: [PATCH 04/21] try fixing workflow yaml too --- .github/workflows/test.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b640d9e..a84b3ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,29 +19,14 @@ jobs: - name: Extras / Count Lines of Source Code run: make extras/cloc - - name: Install requirements - run: make init - - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - # Build [Dev] (and Deploy) - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - name: Build (dev) - if: github.ref == 'refs/heads/dev' - run: pnpm build:dev - - - name: Deploy (dev) [Copy static files over] - if: github.ref == 'refs/heads/dev' - run: rm -rf /var/www/app/* && mv build/* /var/www/app/ - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Lint # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - name: Lint - run: pnpm lint + run: make format - name: Check - run: pnpm check + run: git diff HEAD --quiet # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Build [Prod] (and Upload binary) @@ -52,7 +37,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - RELEASE_TAG=v$(jq -r .version package.json) + RELEASE_TAG=v$(python3 sql/latest_version.py) echo RELEASE_TAG=$RELEASE_TAG test -n "$RELEASE_TAG" # Test that github-cli is working From 33d939cafd687030a6118420c6d72e0bfd9759a4 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 14:23:29 -0500 Subject: [PATCH 05/21] try-except in latest_version.py --- sql/latest_version.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sql/latest_version.py b/sql/latest_version.py index a5214ca..0cd53a6 100644 --- a/sql/latest_version.py +++ b/sql/latest_version.py @@ -11,13 +11,17 @@ SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) -version_csv_path = os.path.join( - SCRIPT_DIR, "version.csv" -) +try: + version_csv_path = os.path.join( + SCRIPT_DIR, "version.csv" + ) -rows = [] -with open(version_csv_path, "r", encoding="utf-8") as _r_file: - reader = csv.reader(_r_file) - rows = list(reader) + rows = [] + with open(version_csv_path, "r", encoding="utf-8") as _r_file: + reader = csv.reader(_r_file) + rows = list(reader) -print(rows[-1][1], end="") + print(rows[-1][15], end="") +except Exception as exc: + # Failed, so we return empty version + pass From 72a4b7517dfe97561bf5c66c74817dcba9d17686 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 14:23:39 -0500 Subject: [PATCH 06/21] black . --- data/process.py | 6 +++--- sql/latest_version.py | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/data/process.py b/data/process.py index ae1f9a2..6e5a725 100644 --- a/data/process.py +++ b/data/process.py @@ -69,7 +69,7 @@ def main(args): - """ Processes the USDA data to get ready for ntdb """ + """Processes the USDA data to get ready for ntdb""" # ----------------- # Process USDA csv @@ -98,7 +98,7 @@ def main(args): # Handle general file # ---------------------- def process(rows, fname): - """ Processes FD_GRP only :O """ + """Processes FD_GRP only :O""" with open(output_files[fname], "w+") as file: writer = csv.writer(file, lineterminator="\n") @@ -109,7 +109,7 @@ def process(rows, fname): # Nutrient defs # ----------------- def process_nutr_def(): - """ Process nutr_def """ + """Process nutr_def""" def process_main(rows): result = [] diff --git a/sql/latest_version.py b/sql/latest_version.py index 0cd53a6..4a34fe8 100644 --- a/sql/latest_version.py +++ b/sql/latest_version.py @@ -12,9 +12,7 @@ SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) try: - version_csv_path = os.path.join( - SCRIPT_DIR, "version.csv" - ) + version_csv_path = os.path.join(SCRIPT_DIR, "version.csv") rows = [] with open(version_csv_path, "r", encoding="utf-8") as _r_file: From 12b0ee97d7c46d8c1061c3ecbb36ec70b36d58f2 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 14:25:23 -0500 Subject: [PATCH 07/21] unset intentional error --- sql/latest_version.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/latest_version.py b/sql/latest_version.py index 4a34fe8..4fe02e6 100644 --- a/sql/latest_version.py +++ b/sql/latest_version.py @@ -19,7 +19,9 @@ reader = csv.reader(_r_file) rows = list(reader) - print(rows[-1][15], end="") + # Print latest version + print(rows[-1][1], end="") + except Exception as exc: - # Failed, so we return empty version + # Failed, so we print empty version pass From 92d59c3da9222401a715a4296a66ff08002001df Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 14:38:24 -0500 Subject: [PATCH 08/21] TEST THIS: updated process.py --- data/process.py | 138 ++++++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/data/process.py b/data/process.py index 6e5a725..ba2c283 100644 --- a/data/process.py +++ b/data/process.py @@ -1,18 +1,20 @@ -# nt-sqlite, an sqlite3 database for nutratracker clients -# Copyright (C) 2019-2020 Shane Jaroch +""" +nt-sqlite, an sqlite3 database for nutratracker clients +Copyright (C) 2019-2020 Shane Jaroch -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" import csv import os @@ -21,10 +23,10 @@ # Check Python version if sys.version_info < (3, 7, 0): - ver = ".".join([str(x) for x in sys.version_info[0:3]]) + _VERSION = ".".join([str(x) for x in sys.version_info[0:3]]) print("ERROR: this requires Python 3.7.0 or later to run") - print("HINT: You're running Python " + ver) - exit(1) + print("HINT: You're running Python " + _VERSION) + sys.exit(1) # change to script's dir os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -55,20 +57,18 @@ # RDAs # -------------------- rdas = {} -with open("rda.csv") as file: +with open("rda.csv", "r", encoding="utf-8") as file: reader = csv.DictReader(file) - rdas = list(reader) - rdas = {int(x["id"]): x for x in rdas} + rdas = {int(x["id"]): x for x in list(reader)} -""" # -------------------- # main method # -------------------- -""" +# TODO: support args input? -def main(args): +def main(): """Processes the USDA data to get ready for ntdb""" # ----------------- @@ -83,9 +83,8 @@ def main(args): for fname in output_files: print(fname) # Open the CSV file - with open(fname) as file: - reader = csv.reader(file) - rows = list(reader) + with open(fname, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) ######################### # Process and write out if fname == "SR-Leg_DB/WEIGHT.csv": @@ -100,8 +99,8 @@ def main(args): def process(rows, fname): """Processes FD_GRP only :O""" - with open(output_files[fname], "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open(output_files[fname], "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(rows) @@ -170,9 +169,8 @@ def process_si(rows): # Main USDA files main_nutr = "SR-Leg_DB/NUTR_DEF.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.DictReader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.DictReader(_file)) rows = process_main(rows) # Add to final solution result.extend(rows) @@ -181,16 +179,15 @@ def process_si(rows): for dir in special_interests_dirs: sub_nutr = f"{dir}/NUTR_DEF.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.DictReader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.DictReader(_file)) rows = process_si(rows) # Add to final solution result.extend(rows) ######################### # Write out result - with open("nt/nutr_def.csv", "w+") as file: + with open("nt/nutr_def.csv", "w+", encoding="utf-8") as _file: fieldnames = list(result[0].keys()) writer = csv.DictWriter(file, fieldnames=fieldnames, lineterminator="\n") writer.writeheader() @@ -201,16 +198,16 @@ def process_si(rows): # Nutrient data # ----------------- def process_nut_data(): - # + """Process nut_data""" + # Prepare the rows result = [] # Main USDA files main_nutr = "SR-Leg_DB/NUT_DATA.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) rows[0].append("cc") # CC, see: Flav_R03-1.pdf # Add to final solution for row in rows: @@ -221,9 +218,8 @@ def process_nut_data(): for dir in special_interests_dirs: sub_nutr = f"{dir}/NUT_DATA.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) # Add to final solution for row in rows[1:]: _row = [None] * 18 @@ -241,8 +237,8 @@ def process_nut_data(): ######################### # Write out result - with open("nt/nut_data.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/nut_data.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(result) @@ -250,7 +246,8 @@ def process_nut_data(): # Food description # ----------------- def process_food_des(): - # + """Process food_des""" + # Prepare the rows result = [] food_ids = set() @@ -258,9 +255,9 @@ def process_food_des(): # Main USDA files main_nutr = "SR-Leg_DB/FOOD_DES.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for i, row in enumerate(rows): if i > 0: @@ -271,9 +268,9 @@ def process_food_des(): for dir in special_interests_dirs: sub_nutr = f"{dir}/FOOD_DES.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: food_id = int(_row[0]) @@ -293,8 +290,8 @@ def process_food_des(): ######################### # Write out result - with open("nt/food_des.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/food_des.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(result) @@ -302,7 +299,8 @@ def process_food_des(): # Data sources # ----------------- def process_data_srcs(): - # + """Process data_srcs""" + # Prepare the rows data_src = [] datsrcln = [] @@ -312,7 +310,9 @@ def process_data_srcs(): main_datsrcln = "SR-Leg_DB/DATSRCLN.csv" print(main_data_src) print(main_datsrcln) - with open(main_data_src) as file_src, open(main_datsrcln) as file_ln: + with open(main_data_src, "r", encoding="utf-8") as file_src, open( + main_datsrcln, "r", encoding="utf-8" + ) as file_ln: reader_src = csv.reader(file_src) data_src_rows = list(reader_src) reader_ln = csv.reader(file_ln) @@ -329,9 +329,9 @@ def process_data_srcs(): # DATA_SRC.csv sub_nutr = f"{dir}/DATA_SRC.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: # Special rules @@ -344,20 +344,20 @@ def process_data_srcs(): # DATASRCLN.csv sub_nutr = f"{dir}/DATSRCLN.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: datsrcln.append(_row) ################################################## # Write serv_desc and serving tables - with open("nt/data_src.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/data_src.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(data_src) - with open("nt/datsrcln.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/datsrcln.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(datsrcln) @@ -365,6 +365,7 @@ def process_data_srcs(): # Weight # ----------------- def process_weight(rows, fname): + """Process weight""" # Unique qualifiers msre_ids = {} @@ -406,15 +407,16 @@ def process_weight(rows, fname): ################################################## # Write serv_desc and serving tables - with open("nt/serv_desc.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/serv_desc.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(serv_desc) - with open("nt/serving.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/serving.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(serving) # # Make script executable if __name__ == "__main__": - main(sys.argv[1:]) + # main(sys.argv[1:]) + main() From cb96f669ccd9512e8c348bb800cd9108718d929c Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sat, 2 Mar 2024 19:14:10 -0500 Subject: [PATCH 09/21] fix process.py --- data/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/process.py b/data/process.py index ba2c283..bd79313 100644 --- a/data/process.py +++ b/data/process.py @@ -189,7 +189,7 @@ def process_si(rows): # Write out result with open("nt/nutr_def.csv", "w+", encoding="utf-8") as _file: fieldnames = list(result[0].keys()) - writer = csv.DictWriter(file, fieldnames=fieldnames, lineterminator="\n") + writer = csv.DictWriter(_file, fieldnames=fieldnames, lineterminator="\n") writer.writeheader() writer.writerows(result) From abe0f4db660212a9bca036ec47791c506c12d0ba Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Mon, 15 Apr 2024 22:45:12 -0400 Subject: [PATCH 10/21] add check-vars --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index d9afdff..62b58db 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,14 @@ clean: ## Clean up leftover bits and stuff from build rm -f sql/*.sqlite rm -f sql/*.sqlite3 +.PHONY: check-vars +check-vars: ## display all computed vars (won't show passed in) + $(info --vars--) + $(foreach v, \ + $(filter-out $(FILTERED_VARS) FILTERED_VARS,$(.VARIABLES)), \ + $(info $(v) = $($(v)))) + @printf '' + .PHONY: extras/cloc extras/cloc: cloc HEAD --exclude-dir=usda.svg From 9244f27638131be8250c4e5dbcf832b8e763c000 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Mon, 15 Apr 2024 23:11:35 -0400 Subject: [PATCH 11/21] update makefile, workflow --- .github/workflows/test.yml | 7 +++---- Makefile | 22 +++++++++------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a84b3ef..08c221e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,13 +37,12 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - RELEASE_TAG=v$(python3 sql/latest_version.py) - echo RELEASE_TAG=$RELEASE_TAG - test -n "$RELEASE_TAG" + RELEASE_TAG=$(make deploy/get-current-db-version) # Test that github-cli is working gh --version gh release list -L 1 - # TODO: enhance this to be: if release_tag > current_prod_tag, deploy + # TODO: enhance this to be: if release_tag > current_prod_tag, deploy. + # Otherwise, can skip this step entirely? gh release view $RELEASE_TAG || echo "PUBLISH=1" >> "$GITHUB_OUTPUT" # yamllint disable rule:line-length diff --git a/Makefile b/Makefile index 62b58db..c07423c 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,13 @@ build: ## Build the release ./sql/build.sh ${DB_VERSION} du -h ${DB_FILE} + +.PHONY: deploy/get-current-db-version +deploy/get-current-db-version: + @test "${DB_VERSION}" + @echo v${DB_VERSION} + + .PHONY: deploy/upload deploy/upload: ## Upload to GitHub releases test -n "${DB_VERSION}" @@ -60,20 +67,13 @@ deploy/upload: ## Upload to GitHub releases .PHONY: deploy/delete deploy/delete: + [[ "$(shell read -e -p 'Really delete v${DB_VERSION}? [y/N]> '; echo $$REPLY)" == [Yy]* ]] gh release delete v${DB_VERSION} git push origin --delete v${DB_VERSION} - git tag -d v${DB_VERSION} REMOTE_HEAD ?= origin/master - -.PHONY: _check-git-up-to-date -_check-git-up-to-date: - git branch --show-current - git fetch - # Check that we are in sync with ${REMOTE_HEAD} - git diff --quiet ${REMOTE_HEAD} - PROJECT_NAME ?= usda-sqlite .PHONY: deploy/install-prod @@ -98,11 +98,7 @@ clean: ## Clean up leftover bits and stuff from build .PHONY: check-vars check-vars: ## display all computed vars (won't show passed in) - $(info --vars--) - $(foreach v, \ - $(filter-out $(FILTERED_VARS) FILTERED_VARS,$(.VARIABLES)), \ - $(info $(v) = $($(v)))) - @printf '' + $(foreach v, $(.VARIABLES), $(if $(filter file, $(origin $(v))), $(info $(v)=$($(v))))) .PHONY: extras/cloc extras/cloc: From 91a4183848b3b61a0bf51a7b5186f0144701f36e Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 11:44:19 -0400 Subject: [PATCH 12/21] lint sql/latest_version.py --- sql/latest_version.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) mode change 100644 => 100755 sql/latest_version.py diff --git a/sql/latest_version.py b/sql/latest_version.py old mode 100644 new mode 100755 index 4fe02e6..0e9aa91 --- a/sql/latest_version.py +++ b/sql/latest_version.py @@ -8,20 +8,30 @@ import csv import os +import sys SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) -try: - version_csv_path = os.path.join(SCRIPT_DIR, "version.csv") - rows = [] - with open(version_csv_path, "r", encoding="utf-8") as _r_file: - reader = csv.reader(_r_file) - rows = list(reader) +def print_version() -> int: + """Prints latest version. Print nothing on error or missing value/file.""" + try: + version_csv_path = os.path.join(SCRIPT_DIR, "version.csv") - # Print latest version - print(rows[-1][1], end="") + # Gather version.CSV into list + rows = [] + with open(version_csv_path, "r", encoding="utf-8") as _r_file: + reader = csv.reader(_r_file) + rows = list(reader) -except Exception as exc: - # Failed, so we print empty version - pass + # Print latest version + print(rows[-1][1]) + return 0 + + except Exception: # pylint: disable=broad-exception-caught + # Failed, so we print empty version + return 1 + + +if __name__ == "__main__": + sys.exit(print_version()) From 810083f1299e13bb403f54a75c57b9ae49c2ecc2 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 11:46:32 -0400 Subject: [PATCH 13/21] fix README title and status badge (image) --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 2c24a6b..35fbc09 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ -*********** +************* usda-sqlite -*********** +************* -.. image:: https://api.travis-ci.com/nutratech/usda-sqlite.svg?branch=master - :target: https://travis-ci.com/github/nutratech/usda-sqlite +.. image:: https://github.com/nutratech/usda-sqlite/actions/workflows/test.yml/badge.svg + :target: https://github.com/nutratech/usda-sqlite/actions/workflows/test.yml Python, SQL and CSV files for setting up portable usda-sqlite database. From c949dc05b6f18cca41f2224a02b7ba30ec483a7d Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 12:06:38 -0400 Subject: [PATCH 14/21] makefile, readme --- Makefile | 51 +++++++++++++++++++++++++++------------------------ README.rst | 29 ++++++++++++++++++----------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index c07423c..c01090e 100644 --- a/Makefile +++ b/Makefile @@ -9,23 +9,6 @@ _help: -# --------------------------------------- -# Install requirements -# --------------------------------------- - -.PHONY: init -init: ## Install requirements (w/o --frozen-lockfile) - # Check version - [[ "$(shell pnpm --version)" =~ "8." ]] - # Remove old install - rm -rf node_modules/ - # Install requirements - pnpm install - # Sync svelte kit (optional) - pnpm svelte-kit sync - - - # --------------------------------------- # Format # --------------------------------------- @@ -33,6 +16,7 @@ init: ## Install requirements (w/o --frozen-lockfile) .PHONY: format format: ## format SQL with pg_format # TODO: what about import.sql? It gets formatted too ugly + # TODO: what about Python files? pg_format -L -s 2 -w 100 sql/tables.sql >sql/tables.fmt.sql mv sql/tables.fmt.sql sql/tables.sql\ @@ -43,13 +27,24 @@ format: ## format SQL with pg_format # --------------------------------------- DB_VERSION ?= $(shell python3 sql/latest_version.py) -DB_FILE ?= sql/dist/usda.sqlite3-${DB_VERSION}.tar.xz +DB_FILE ?= sql/usda.sqlite3 +DB_XZ_FILE ?= sql/dist/usda.sqlite3-${DB_VERSION}.tar.xz .PHONY: build build: clean -build: ## Build the release +build: ## Build the release (compressed XZ file) ./sql/build.sh ${DB_VERSION} - du -h ${DB_FILE} + du -h ${DB_XZ_FILE} + +.PHONY: test +test: ## Test the SQL database with basic queries + test -f ${DB_FILE} + sqlite3 ${DB_FILE} ".tables" + sqlite3 ${DB_FILE} "\ + SELECT * FROM nutr_def WHERE id=328; \ + SELECT long_desc FROM food_des WHERE id=9050; \ + SELECT * FROM version; \ + " .PHONY: deploy/get-current-db-version @@ -61,9 +56,9 @@ deploy/get-current-db-version: .PHONY: deploy/upload deploy/upload: ## Upload to GitHub releases test -n "${DB_VERSION}" - test -f ${DB_FILE} + test -f ${DB_XZ_FILE} gh release create v${DB_VERSION} --generate-notes - gh release upload v${DB_VERSION} ${DB_FILE} + gh release upload v${DB_VERSION} ${DB_XZ_FILE} .PHONY: deploy/delete deploy/delete: @@ -76,14 +71,22 @@ deploy/delete: REMOTE_HEAD ?= origin/master PROJECT_NAME ?= usda-sqlite +.PHONY: _check-git-up-to-date +_check-git-up-to-date: + git branch --show-current + git fetch + # Check that we are in sync with ${REMOTE_HEAD} + git diff --quiet ${REMOTE_HEAD} + + .PHONY: deploy/install-prod deploy/install-prod: _check-git-up-to-date deploy/install-prod: ## Install (on prod VPS) # Check the version string was extracted from package.json test -n "${DB_VERSION}" # Download ${DB_VERSION} - curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${DB_VERSION}/${DB_FILE} - tar xf ${DB_FILE} + curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${DB_VERSION}/${DB_XZ_FILE} + tar xf ${DB_XZ_FILE} diff --git a/README.rst b/README.rst index 35fbc09..a3e006e 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,8 @@ Building the database bash setup.sh python3 process.py -3. If you are committing database changes, add a line to :code:`sql/version.csv` (e.g. :code:`id=3` is the latest in this case), +3. If you are committing database changes, add a line to + :code:`sql/version.csv` (e.g. :code:`id=3` is the latest in this case). +-----+----------+-----------------------------------+ | id | version | created | @@ -44,7 +45,7 @@ Building the database 4. i. (Optional) Enforce foreign keys with your ``~/.sqliterc`` file, -:: +.. code-block:: text .headers on .mode column @@ -54,21 +55,27 @@ Building the database .. code-block:: bash - cd ../sql - ./build.sh X.X.X # e.g. 0.0.8 + make build -5. Verify the tables (again inside the SQL shell :code:`sqlite3 usda.sqlite3`), +5. Verify the tables (inside an SQL shell: :code:`sqlite3 sql/usda.sqlite3`), .. code-block:: sql - .tables - SELECT * FROM nutr_def WHERE id=328; - SELECT long_desc FROM food_des WHERE id=9050; - SELECT * FROM version; - .exit + .tables -6. If everything looks good, upload compressed :code:`dist/nutra-X.X.X.db.tar.xz` file to binary host (bitbucket files). + SELECT * FROM nutr_def WHERE id=328; + SELECT long_desc FROM food_des WHERE id=9050; + SELECT * FROM version; + + .exit + +6. If everything looks good, upload compressed + :code:`dist/nutra-X.X.X.db.tar.xz` file to binary host. + +.. code-block:: bash + + make deploy/upload Tables (Relational Design) From 905b3a2653b294bdf4f2931a91660c607bba3ed3 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 12:10:41 -0400 Subject: [PATCH 15/21] del deploy/install-prod and _check-git-up-to-date --- Makefile | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index c01090e..1bb91a4 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL=/bin/bash # NOTE: must put a character and two pound "\t##" to show up in this list. Keep it brief! IGNORE_ME .PHONY: _help _help: - @grep -h "##" $(MAKEFILE_LIST) | grep -v IGNORE_ME | sed -e 's/##//' | column -t -s $$'\t' + @grep -h "##" $(MAKEFILE_LIST) | grep -v IGNORE_ME | grep -v ^# | sed -e 's/##//' | column -t -s $$'\t' @@ -18,7 +18,7 @@ format: ## format SQL with pg_format # TODO: what about import.sql? It gets formatted too ugly # TODO: what about Python files? pg_format -L -s 2 -w 100 sql/tables.sql >sql/tables.fmt.sql - mv sql/tables.fmt.sql sql/tables.sql\ + mv sql/tables.fmt.sql sql/tables.sql @@ -33,6 +33,7 @@ DB_XZ_FILE ?= sql/dist/usda.sqlite3-${DB_VERSION}.tar.xz .PHONY: build build: clean build: ## Build the release (compressed XZ file) + test "${DB_VERSION}" ./sql/build.sh ${DB_VERSION} du -h ${DB_XZ_FILE} @@ -68,27 +69,6 @@ deploy/delete: - git tag -d v${DB_VERSION} -REMOTE_HEAD ?= origin/master -PROJECT_NAME ?= usda-sqlite - -.PHONY: _check-git-up-to-date -_check-git-up-to-date: - git branch --show-current - git fetch - # Check that we are in sync with ${REMOTE_HEAD} - git diff --quiet ${REMOTE_HEAD} - - -.PHONY: deploy/install-prod -deploy/install-prod: _check-git-up-to-date -deploy/install-prod: ## Install (on prod VPS) - # Check the version string was extracted from package.json - test -n "${DB_VERSION}" - # Download ${DB_VERSION} - curl -sSLO https://github.com/nutratech/${PROJECT_NAME}/releases/download/${DB_VERSION}/${DB_XZ_FILE} - tar xf ${DB_XZ_FILE} - - # --------------------------------------- # Clean & extras @@ -104,5 +84,5 @@ check-vars: ## display all computed vars (won't show passed in) $(foreach v, $(.VARIABLES), $(if $(filter file, $(origin $(v))), $(info $(v)=$($(v))))) .PHONY: extras/cloc -extras/cloc: +extras/cloc: ## count lines of code cloc HEAD --exclude-dir=usda.svg From ef9a0591f572b94041a4fd1b8e612bc4f7eb642e Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 12:25:10 -0400 Subject: [PATCH 16/21] update readme --- README.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index a3e006e..7b457bd 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,7 @@ See CLI: https://github.com/nutratech/cli See nt-sqlite: https://github.com/nutratech/nt-sqlite + Building the database ######################### @@ -30,6 +31,7 @@ Building the database bash setup.sh python3 process.py + 3. If you are committing database changes, add a line to :code:`sql/version.csv` (e.g. :code:`id=3` is the latest in this case). @@ -43,7 +45,8 @@ Building the database | 3 | 0.0.2 | Thu 06 Aug 2020 09:21:39 AM EDT | +-----+----------+-----------------------------------+ -4. i. (Optional) Enforce foreign keys with your ``~/.sqliterc`` file, + +4. i. *(Optional)* Enforce FKs by copying this to your ``~/.sqliterc`` file. .. code-block:: text @@ -51,24 +54,19 @@ Building the database .mode column PRAGMA foreign_keys = 1; -4. ii. Create the database with +4. ii. Create the database. .. code-block:: bash make build -5. Verify the tables (inside an SQL shell: :code:`sqlite3 sql/usda.sqlite3`), +5. Verify the tables. .. code-block:: sql - .tables + make test - SELECT * FROM nutr_def WHERE id=328; - SELECT long_desc FROM food_des WHERE id=9050; - SELECT * FROM version; - - .exit 6. If everything looks good, upload compressed :code:`dist/nutra-X.X.X.db.tar.xz` file to binary host. @@ -78,6 +76,7 @@ Building the database make deploy/upload + Tables (Relational Design) ########################## From a96acca607cacad792675e67e4e6bb6c1c8c6486 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 12:26:44 -0400 Subject: [PATCH 17/21] add make docs target --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 1bb91a4..cfb9175 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,10 @@ test: ## Test the SQL database with basic queries SELECT * FROM version; \ " +.PHONY: docs +docs: ## Build the relational SVG diagram + ./docs/sqleton.sh + .PHONY: deploy/get-current-db-version deploy/get-current-db-version: From d8a4732a03601a9d66c6d24b9a56d2792a687cc5 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Thu, 18 Apr 2024 12:29:20 -0400 Subject: [PATCH 18/21] adjust sections in Makefile --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cfb9175..ad9a786 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ format: ## format SQL with pg_format # --------------------------------------- -# Build & install +# Build, test, and docs # --------------------------------------- DB_VERSION ?= $(shell python3 sql/latest_version.py) @@ -52,6 +52,11 @@ docs: ## Build the relational SVG diagram ./docs/sqleton.sh + +# --------------------------------------- +# Deploy +# --------------------------------------- + .PHONY: deploy/get-current-db-version deploy/get-current-db-version: @test "${DB_VERSION}" From bf29771ad96f3b1a66667a34fbfcc0c823429358 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sun, 21 Apr 2024 22:02:31 -0400 Subject: [PATCH 19/21] remove old travis CI config file --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6118eff..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -dist: xenial -os: ['linux'] -language: python -python: -- '3.7' -script: -- bash data/setup.sh -- python data/process.py -- cd sql && sqlite3 usda.sqlite ".read init.sql" From 8f27bfe5e6d0f9b74f6ced72f8dcc425771a57fc Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Sun, 21 Apr 2024 22:03:24 -0400 Subject: [PATCH 20/21] remove TODO --- TODO | 1 - 1 file changed, 1 deletion(-) delete mode 100644 TODO diff --git a/TODO b/TODO deleted file mode 100644 index 1aebcea..0000000 --- a/TODO +++ /dev/null @@ -1 +0,0 @@ -automatically publish GH release if version is incremented From 8324bf302bc2417dbee8f5c7280acaaef620fd65 Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Mon, 22 Apr 2024 21:43:15 -0400 Subject: [PATCH 21/21] remove format.sh; remove blank line in py file --- sql/format.sh | 7 ------- sql/latest_version.py | 1 - 2 files changed, 8 deletions(-) delete mode 100755 sql/format.sh diff --git a/sql/format.sh b/sql/format.sh deleted file mode 100755 index 8cc5c17..0000000 --- a/sql/format.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")" - -# TODO: what about import.sql? It gets formatted too ugly -pg_format -L -s 2 -w 100 tables.sql >tables.fmt.sql -mv tables.fmt.sql tables.sql diff --git a/sql/latest_version.py b/sql/latest_version.py index 0e9aa91..341bdd4 100755 --- a/sql/latest_version.py +++ b/sql/latest_version.py @@ -5,7 +5,6 @@ @author: shane """ - import csv import os import sys