diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..08c221e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,59 @@ +--- +name: ci + +'on': + push: + branches: + - '**' + +permissions: + contents: write + +jobs: + ci: + runs-on: [self-hosted, dev] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extras / Count Lines of Source Code + run: make extras/cloc + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Lint + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + - name: Lint + run: make format + + - name: Check + run: git diff HEAD --quiet + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Build [Prod] (and Upload binary) + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + - name: Check release version + id: check-release-version + if: github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ github.token }} + run: | + RELEASE_TAG=$(make deploy/get-current-db-version) + # Test that github-cli is working + gh --version + gh release list -L 1 + # TODO: enhance this to be: if release_tag > current_prod_tag, deploy. + # Otherwise, can skip this step entirely? + gh release view $RELEASE_TAG || echo "PUBLISH=1" >> "$GITHUB_OUTPUT" + + # yamllint disable rule:line-length + - name: Build (production release) + if: github.ref == 'refs/heads/master' && steps.check-release-version.outputs.PUBLISH + env: + GH_TOKEN: ${{ github.token }} + run: set -o pipefail; make build + + - name: Upload artifacts (production release) + if: github.ref == 'refs/heads/master' && steps.check-release-version.outputs.PUBLISH + env: + GH_TOKEN: ${{ github.token }} + run: set -o pipefail; make deploy/upload diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6118eff..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -dist: xenial -os: ['linux'] -language: python -python: -- '3.7' -script: -- bash data/setup.sh -- python data/process.py -- cd sql && sqlite3 usda.sqlite ".read init.sql" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ad9a786 --- /dev/null +++ b/Makefile @@ -0,0 +1,97 @@ +SHELL=/bin/bash + +.DEFAULT_GOAL := _help + +# NOTE: must put a character and two pound "\t##" to show up in this list. Keep it brief! IGNORE_ME +.PHONY: _help +_help: + @grep -h "##" $(MAKEFILE_LIST) | grep -v IGNORE_ME | grep -v ^# | sed -e 's/##//' | column -t -s $$'\t' + + + +# --------------------------------------- +# Format +# --------------------------------------- + +.PHONY: format +format: ## format SQL with pg_format + # TODO: what about import.sql? It gets formatted too ugly + # TODO: what about Python files? + pg_format -L -s 2 -w 100 sql/tables.sql >sql/tables.fmt.sql + mv sql/tables.fmt.sql sql/tables.sql + + + +# --------------------------------------- +# Build, test, and docs +# --------------------------------------- + +DB_VERSION ?= $(shell python3 sql/latest_version.py) +DB_FILE ?= sql/usda.sqlite3 +DB_XZ_FILE ?= sql/dist/usda.sqlite3-${DB_VERSION}.tar.xz + +.PHONY: build +build: clean +build: ## Build the release (compressed XZ file) + test "${DB_VERSION}" + ./sql/build.sh ${DB_VERSION} + du -h ${DB_XZ_FILE} + +.PHONY: test +test: ## Test the SQL database with basic queries + test -f ${DB_FILE} + sqlite3 ${DB_FILE} ".tables" + sqlite3 ${DB_FILE} "\ + SELECT * FROM nutr_def WHERE id=328; \ + SELECT long_desc FROM food_des WHERE id=9050; \ + SELECT * FROM version; \ + " + +.PHONY: docs +docs: ## Build the relational SVG diagram + ./docs/sqleton.sh + + + +# --------------------------------------- +# Deploy +# --------------------------------------- + +.PHONY: deploy/get-current-db-version +deploy/get-current-db-version: + @test "${DB_VERSION}" + @echo v${DB_VERSION} + + +.PHONY: deploy/upload +deploy/upload: ## Upload to GitHub releases + test -n "${DB_VERSION}" + test -f ${DB_XZ_FILE} + gh release create v${DB_VERSION} --generate-notes + gh release upload v${DB_VERSION} ${DB_XZ_FILE} + +.PHONY: deploy/delete +deploy/delete: + [[ "$(shell read -e -p 'Really delete v${DB_VERSION}? [y/N]> '; echo $$REPLY)" == [Yy]* ]] + gh release delete v${DB_VERSION} + git push origin --delete v${DB_VERSION} + - git tag -d v${DB_VERSION} + + + +# --------------------------------------- +# Clean & extras +# --------------------------------------- + +.PHONY: clean +clean: ## Clean up leftover bits and stuff from build + rm -f sql/*.sqlite + rm -f sql/*.sqlite3 + +.PHONY: check-vars +check-vars: ## display all computed vars (won't show passed in) + $(foreach v, $(.VARIABLES), $(if $(filter file, $(origin $(v))), $(info $(v)=$($(v))))) + +.PHONY: extras/cloc +extras/cloc: ## count lines of code + cloc HEAD --exclude-dir=usda.svg diff --git a/README.rst b/README.rst index 2c24a6b..7b457bd 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ -*********** +************* usda-sqlite -*********** +************* -.. image:: https://api.travis-ci.com/nutratech/usda-sqlite.svg?branch=master - :target: https://travis-ci.com/github/nutratech/usda-sqlite +.. image:: https://github.com/nutratech/usda-sqlite/actions/workflows/test.yml/badge.svg + :target: https://github.com/nutratech/usda-sqlite/actions/workflows/test.yml Python, SQL and CSV files for setting up portable usda-sqlite database. @@ -12,6 +12,7 @@ See CLI: https://github.com/nutratech/cli See nt-sqlite: https://github.com/nutratech/nt-sqlite + Building the database ######################### @@ -30,7 +31,9 @@ Building the database bash setup.sh python3 process.py -3. If you are committing database changes, add a line to :code:`sql/version.csv` (e.g. :code:`id=3` is the latest in this case), + +3. If you are committing database changes, add a line to + :code:`sql/version.csv` (e.g. :code:`id=3` is the latest in this case). +-----+----------+-----------------------------------+ | id | version | created | @@ -42,33 +45,36 @@ Building the database | 3 | 0.0.2 | Thu 06 Aug 2020 09:21:39 AM EDT | +-----+----------+-----------------------------------+ -4. i. (Optional) Enforce foreign keys with your ``~/.sqliterc`` file, -:: +4. i. *(Optional)* Enforce FKs by copying this to your ``~/.sqliterc`` file. + +.. code-block:: text .headers on .mode column PRAGMA foreign_keys = 1; -4. ii. Create the database with +4. ii. Create the database. .. code-block:: bash - cd ../sql - ./build.sh X.X.X # e.g. 0.0.8 + make build -5. Verify the tables (again inside the SQL shell :code:`sqlite3 usda.sqlite3`), +5. Verify the tables. .. code-block:: sql - .tables - SELECT * FROM nutr_def WHERE id=328; - SELECT long_desc FROM food_des WHERE id=9050; - SELECT * FROM version; - .exit + make test + + +6. If everything looks good, upload compressed + :code:`dist/nutra-X.X.X.db.tar.xz` file to binary host. + +.. code-block:: bash + + make deploy/upload -6. If everything looks good, upload compressed :code:`dist/nutra-X.X.X.db.tar.xz` file to binary host (bitbucket files). Tables (Relational Design) diff --git a/TODO b/TODO deleted file mode 100644 index 1aebcea..0000000 --- a/TODO +++ /dev/null @@ -1 +0,0 @@ -automatically publish GH release if version is incremented diff --git a/data/process.py b/data/process.py index ae1f9a2..bd79313 100644 --- a/data/process.py +++ b/data/process.py @@ -1,18 +1,20 @@ -# nt-sqlite, an sqlite3 database for nutratracker clients -# Copyright (C) 2019-2020 Shane Jaroch +""" +nt-sqlite, an sqlite3 database for nutratracker clients +Copyright (C) 2019-2020 Shane Jaroch -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" import csv import os @@ -21,10 +23,10 @@ # Check Python version if sys.version_info < (3, 7, 0): - ver = ".".join([str(x) for x in sys.version_info[0:3]]) + _VERSION = ".".join([str(x) for x in sys.version_info[0:3]]) print("ERROR: this requires Python 3.7.0 or later to run") - print("HINT: You're running Python " + ver) - exit(1) + print("HINT: You're running Python " + _VERSION) + sys.exit(1) # change to script's dir os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -55,21 +57,19 @@ # RDAs # -------------------- rdas = {} -with open("rda.csv") as file: +with open("rda.csv", "r", encoding="utf-8") as file: reader = csv.DictReader(file) - rdas = list(reader) - rdas = {int(x["id"]): x for x in rdas} + rdas = {int(x["id"]): x for x in list(reader)} -""" # -------------------- # main method # -------------------- -""" +# TODO: support args input? -def main(args): - """ Processes the USDA data to get ready for ntdb """ +def main(): + """Processes the USDA data to get ready for ntdb""" # ----------------- # Process USDA csv @@ -83,9 +83,8 @@ def main(args): for fname in output_files: print(fname) # Open the CSV file - with open(fname) as file: - reader = csv.reader(file) - rows = list(reader) + with open(fname, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) ######################### # Process and write out if fname == "SR-Leg_DB/WEIGHT.csv": @@ -98,10 +97,10 @@ def main(args): # Handle general file # ---------------------- def process(rows, fname): - """ Processes FD_GRP only :O """ + """Processes FD_GRP only :O""" - with open(output_files[fname], "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open(output_files[fname], "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(rows) @@ -109,7 +108,7 @@ def process(rows, fname): # Nutrient defs # ----------------- def process_nutr_def(): - """ Process nutr_def """ + """Process nutr_def""" def process_main(rows): result = [] @@ -170,9 +169,8 @@ def process_si(rows): # Main USDA files main_nutr = "SR-Leg_DB/NUTR_DEF.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.DictReader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.DictReader(_file)) rows = process_main(rows) # Add to final solution result.extend(rows) @@ -181,18 +179,17 @@ def process_si(rows): for dir in special_interests_dirs: sub_nutr = f"{dir}/NUTR_DEF.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.DictReader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.DictReader(_file)) rows = process_si(rows) # Add to final solution result.extend(rows) ######################### # Write out result - with open("nt/nutr_def.csv", "w+") as file: + with open("nt/nutr_def.csv", "w+", encoding="utf-8") as _file: fieldnames = list(result[0].keys()) - writer = csv.DictWriter(file, fieldnames=fieldnames, lineterminator="\n") + writer = csv.DictWriter(_file, fieldnames=fieldnames, lineterminator="\n") writer.writeheader() writer.writerows(result) @@ -201,16 +198,16 @@ def process_si(rows): # Nutrient data # ----------------- def process_nut_data(): - # + """Process nut_data""" + # Prepare the rows result = [] # Main USDA files main_nutr = "SR-Leg_DB/NUT_DATA.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) rows[0].append("cc") # CC, see: Flav_R03-1.pdf # Add to final solution for row in rows: @@ -221,9 +218,8 @@ def process_nut_data(): for dir in special_interests_dirs: sub_nutr = f"{dir}/NUT_DATA.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + rows = list(csv.reader(_file)) # Add to final solution for row in rows[1:]: _row = [None] * 18 @@ -241,8 +237,8 @@ def process_nut_data(): ######################### # Write out result - with open("nt/nut_data.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/nut_data.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(result) @@ -250,7 +246,8 @@ def process_nut_data(): # Food description # ----------------- def process_food_des(): - # + """Process food_des""" + # Prepare the rows result = [] food_ids = set() @@ -258,9 +255,9 @@ def process_food_des(): # Main USDA files main_nutr = "SR-Leg_DB/FOOD_DES.csv" print(main_nutr) - with open(main_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(main_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for i, row in enumerate(rows): if i > 0: @@ -271,9 +268,9 @@ def process_food_des(): for dir in special_interests_dirs: sub_nutr = f"{dir}/FOOD_DES.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: food_id = int(_row[0]) @@ -293,8 +290,8 @@ def process_food_des(): ######################### # Write out result - with open("nt/food_des.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/food_des.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(result) @@ -302,7 +299,8 @@ def process_food_des(): # Data sources # ----------------- def process_data_srcs(): - # + """Process data_srcs""" + # Prepare the rows data_src = [] datsrcln = [] @@ -312,7 +310,9 @@ def process_data_srcs(): main_datsrcln = "SR-Leg_DB/DATSRCLN.csv" print(main_data_src) print(main_datsrcln) - with open(main_data_src) as file_src, open(main_datsrcln) as file_ln: + with open(main_data_src, "r", encoding="utf-8") as file_src, open( + main_datsrcln, "r", encoding="utf-8" + ) as file_ln: reader_src = csv.reader(file_src) data_src_rows = list(reader_src) reader_ln = csv.reader(file_ln) @@ -329,9 +329,9 @@ def process_data_srcs(): # DATA_SRC.csv sub_nutr = f"{dir}/DATA_SRC.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: # Special rules @@ -344,20 +344,20 @@ def process_data_srcs(): # DATASRCLN.csv sub_nutr = f"{dir}/DATSRCLN.csv" print(sub_nutr) - with open(sub_nutr) as file: - reader = csv.reader(file) - rows = list(reader) + with open(sub_nutr, "r", encoding="utf-8") as _file: + _reader = csv.reader(_file) + rows = list(_reader) # Add to final solution for _row in rows[1:]: datsrcln.append(_row) ################################################## # Write serv_desc and serving tables - with open("nt/data_src.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/data_src.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(data_src) - with open("nt/datsrcln.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/datsrcln.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(datsrcln) @@ -365,6 +365,7 @@ def process_data_srcs(): # Weight # ----------------- def process_weight(rows, fname): + """Process weight""" # Unique qualifiers msre_ids = {} @@ -406,15 +407,16 @@ def process_weight(rows, fname): ################################################## # Write serv_desc and serving tables - with open("nt/serv_desc.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/serv_desc.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(serv_desc) - with open("nt/serving.csv", "w+") as file: - writer = csv.writer(file, lineterminator="\n") + with open("nt/serving.csv", "w+", encoding="utf-8") as _file: + writer = csv.writer(_file, lineterminator="\n") writer.writerows(serving) # # Make script executable if __name__ == "__main__": - main(sys.argv[1:]) + # main(sys.argv[1:]) + main() diff --git a/sql/format-sql.sh b/sql/format-sql.sh deleted file mode 100755 index 32b0965..0000000 --- a/sql/format-sql.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")" - -pg_format -s 2 tables.sql -o tables.sql -# pg_format -s 2 import.sql -o import.sql diff --git a/sql/latest_version.py b/sql/latest_version.py new file mode 100755 index 0000000..341bdd4 --- /dev/null +++ b/sql/latest_version.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Sat Mar 2 12:32:45 2024 + +@author: shane +""" +import csv +import os +import sys + +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def print_version() -> int: + """Prints latest version. Print nothing on error or missing value/file.""" + try: + version_csv_path = os.path.join(SCRIPT_DIR, "version.csv") + + # Gather version.CSV into list + rows = [] + with open(version_csv_path, "r", encoding="utf-8") as _r_file: + reader = csv.reader(_r_file) + rows = list(reader) + + # Print latest version + print(rows[-1][1]) + return 0 + + except Exception: # pylint: disable=broad-exception-caught + # Failed, so we print empty version + return 1 + + +if __name__ == "__main__": + sys.exit(print_version()) diff --git a/sql/tables.sql b/sql/tables.sql index 9196e3c..3b135ae 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -13,8 +13,11 @@ -- -- You should have received a copy of the GNU General Public License -- along with this program. If not, see . - -CREATE TABLE version( id integer PRIMARY KEY AUTOINCREMENT, version text NOT NULL, created timestamp DEFAULT CURRENT_TIMESTAMP, notes text +CREATE TABLE version ( + id integer PRIMARY KEY AUTOINCREMENT, + version text NOT NULL, + created timestamp DEFAULT CURRENT_TIMESTAMP, + notes text ); CREATE TABLE nutr_def ( @@ -44,10 +47,10 @@ CREATE TABLE food_des ( ref_desc text, refuse int, sci_name text, - n_factor FLOAT, - pro_factor FLOAT, - fat_factor FLOAT, - cho_factor FLOAT, + n_factor float, + pro_factor float, + fat_factor float, + cho_factor float, FOREIGN KEY (fdgrp_id) REFERENCES fdgrp (id) ); @@ -143,4 +146,3 @@ CREATE TABLE serving ( FOREIGN KEY (food_id) REFERENCES food_des (id), FOREIGN KEY (msre_id) REFERENCES serv_desc (id) ); -