diff --git a/hoki/constants.py b/hoki/constants.py index 956700e4..34c00e34 100644 --- a/hoki/constants.py +++ b/hoki/constants.py @@ -40,6 +40,8 @@ "imf135_100", "imf135_300", "imf135all_100", "imf170_100", "imf170_300"] +# wavelengths at which spectra are given [angstrom] +BPASS_WAVELENGTHS = np.arange(1, 100001) # Create a deprecation warning when using dummy_dict dummy_dict = {'timestep': 0, 'age': 1, 'log(R1)': 2, 'log(T1)': 3, 'log(L1)': 4, 'M1': 5, 'He_core1': 6, 'CO_core1': 7, diff --git a/hoki/csp/tests/test_spectra.py b/hoki/csp/tests/test_spectra.py index d111fd71..eb7da2a7 100644 --- a/hoki/csp/tests/test_spectra.py +++ b/hoki/csp/tests/test_spectra.py @@ -13,7 +13,6 @@ from hoki.csp.spectra import CSPSpectra from hoki.load import model_output from hoki.constants import BPASS_NUM_METALLICITIES -import itertools data_path = pkg_resources.resource_filename('hoki', 'data') # Load Test spectra. Not an actual spectra. diff --git a/hoki/data_compilers.py b/hoki/data_compilers.py index 69452770..9cf931e8 100644 --- a/hoki/data_compilers.py +++ b/hoki/data_compilers.py @@ -1,14 +1,14 @@ """ -Author: Max Briel & Heloise Stevance +Author: Heloise Stevance, Max Briel, & Martin Glatzle Objects and pipelines that compile BPASS data files into more convenient, more pythonic data types """ - +import abc import numpy as np - from hoki.constants import (DEFAULT_BPASS_VERSION, MODELS_PATH, OUTPUTS_PATH, dummy_dicts, - BPASS_IMFS, BPASS_METALLICITIES) + BPASS_IMFS, BPASS_METALLICITIES, + BPASS_WAVELENGTHS, BPASS_TIME_BINS) import hoki.load import pandas as pd import re @@ -17,6 +17,120 @@ from hoki.utils.exceptions import HokiFatalError, HokiUserWarning, HokiFormatError, HokiKeyError from hoki.utils.hoki_object import HokiObject +class _CompilerBase(abc.ABC): + def __init__(self, input_folder, output_folder, imf, binary=True, + verbose=False): + if verbose: + _print_welcome() + + # Check population type + star = "bin" if binary else "sin" + + # check IMF key + if imf not in BPASS_IMFS: + raise HokiKeyError( + f"{imf} is not a BPASS IMF. Please select a correct IMF.") + + # Setup the numpy output + output = np.empty(self._shape(), dtype=np.float64) + + # loop over all the metallicities and load all the spectra + for num, metallicity in enumerate(BPASS_METALLICITIES): + print_progress_bar(num, 12) + # Check if file exists + assert isfile(f"{input_folder}/spectra-{star}-{imf}.{metallicity}.dat"),\ + "HOKI ERROR: This file does not exist, or its path is incorrect." + output[num] = self._load_single( + f"{input_folder}/{self._input_name()}-{star}-{imf}.{metallicity}.dat" + ) + + np.save(f"{output_folder}/{self._output_name()}-{star}-{imf}", output) + # pickle the datafile + self.output = output + print( + f"Compiled data stored in {output_folder} as '{self._output_name()}-{star}-{imf}.npy'") + if verbose: + _print_exit_message() + return + + @abc.abstractmethod + def _input_name(self): + return + + @abc.abstractmethod + def _output_name(self): + return + + @abc.abstractmethod + def _shape(self): + return + + @abc.abstractmethod + def _load_single(self, path): + return + +class SpectraCompiler(_CompilerBase): + """ + Pipeline to load the BPASS spectra txt files and save them as a 3D + `numpy.ndarray` binary file. + + Attributes + output : `numpy.ndarray` (N_Z, N_age, N_lam) [(metallicity, log_age, wavelength)] + ---------- + A 3D numpy array containing all the BPASS spectra for a specific imf + and binary or single star population. + Usage: spectra[1][2][1000] + (gives L_\\odot for Z=0.0001 and log_age=6.2 at 999 Angstrom) + """ + + def _input_name(self): + return "spectra" + + def _output_name(self): + return "all_spectra" + + def _shape(self): + return ( + len(BPASS_METALLICITIES), + len(BPASS_TIME_BINS), + len(BPASS_WAVELENGTHS) + ) + + def _load_single(self, path): + return np.loadtxt(path).T[1:, :] + + +class EmissivityCompiler(_CompilerBase): + """ + Pipeline to load the BPASS ionizing txt files and save them as a 3D + `numpy.ndarray` binary file. + + Attributes + ---------- + output : `numpy.ndarray` (N_Z, N_age, 4) [(metallicity, log_age, band)] + A 3D numpy array containing all the BPASS emissivities (Nion [1/s], + L_Halpha [ergs/s], L_FUV [ergs/s/A], L_NUV [ergs/s/A]) for a specific + imf and binary or single star population. + Usage: spectra[1][2][0] + (gives Nion for Z=0.0001 and log_age=6.2) + """ + def _input_name(self): + return "ionizing" + + def _output_name(self): + return "all_ionizing" + + def _shape(self): + return ( + len(BPASS_METALLICITIES), + len(BPASS_TIME_BINS), + 4 + ) + + def _load_single(self, path): + return np.loadtxt(path)[:, 1:] + + class ModelDataCompiler(HokiObject): """ Given a list of metalicities, a list of valid BPASS model attributes (in the dummy array), chosen types of model @@ -202,81 +316,6 @@ def _select_input_files(z_list, directory=OUTPUTS_PATH, return input_file_list -class SpectraCompiler(): - """ - Pipeline to load the BPASS spectra txt files and save them as a 3D - `numpy.ndarray` binary file. - - Attributes - ---------- - spectra : `numpy.ndarray` (13, 51, 100000) [(metallicity, log_age, wavelength)] - A 3D numpy array containing all the BPASS spectra for a specific imf - and binary or single star population. - Usage: spectra[1][2][1000] - (gives L_\\odot for Z=0.0001 and log_age=6.2 at 999 Angstrom) - """ - - def __init__(self, spectra_folder, output_folder, imf, binary=True, verbose=False): - """ - Input - ----- - spectra_folder : `str` - Path to the folder containing the spectra of the given imf. - - output_folder : `str` - Path to the folder, where to output the pickled pandas.DataFrame - - imf : `str` - BPASS IMF Identifiers - The accepted IMF identifiers are: - - `"imf_chab100"` - - `"imf_chab300"` - - `"imf100_100"` - - `"imf100_300"` - - `"imf135_100"` - - `"imf135_300"` - - `"imfall_300"` - - `"imf170_100"` - - `"imf170_300"` - - binary : `bool` - If `True`, loads the binary files. Otherwise, just loads single stars. - Default=True - - verbose : `bool` - If `True` prints out extra information for the user. - Default=False - """ - if verbose: - _print_welcome() - - # Check population type - star = "bin" if binary else "sin" - - # check IMF key - if imf not in BPASS_IMFS: - raise HokiKeyError( - f"{imf} is not a BPASS IMF. Please select a correct IMF.") - - # Setup the numpy output - spectra = np.empty((13, 51, 100000), dtype=np.float64) - - # loop over all the metallicities and load all the spectra - for num, metallicity in enumerate(BPASS_METALLICITIES): - print_progress_bar(num, 12) - assert isfile(f"{spectra_folder}/spectra-{star}-{imf}.{metallicity}.dat"),\ - "HOKI ERROR: This file does not exist, or its path is incorrect." - spectra[num] = np.loadtxt(f"{spectra_folder}/spectra-{star}-{imf}.{metallicity}.dat").T[1:, :] - - # pickle the datafile - np.save(f"{output_folder}/all_spectra-{star}-{imf}", spectra) - self.spectra = spectra - print( - f"Spectra file stored in {output_folder} as 'all_spectra-{star}-{imf}.npy'") - if verbose: - _print_exit_message() - - def _print_welcome(): print('*************************************************') @@ -285,11 +324,10 @@ def _print_welcome(): print("\n\nThis may take a while ;)\nGo get yourself a cup of tea, sit back and relax\nI'm working for you boo!") print( - "\nNOTE: The progress bar doesn't move smoothly - it might accelerate or slow down - it's perfectly normal :D") + "\nNOTE: The progress bar doesn't move smoothly - it might accelerate or slow down - it's perfectly normal :D") def _print_exit_message(): - print('\n\n\n*************************************************') print('******* JOB DONE! HAPPY SCIENCING! ******') print('*************************************************') diff --git a/hoki/load.py b/hoki/load.py index ce3587c0..aba79e16 100755 --- a/hoki/load.py +++ b/hoki/load.py @@ -571,10 +571,82 @@ def spectra_all_z(data_path, imf, binary=True): spec = hoki.data_compilers.SpectraCompiler( data_path, data_path, imf, binary=binary ) - spectra = spec.spectra + spectra = spec.output return spectra +def emissivities_all_z(data_path, imf, binary=True): + """ + Load all BPASS emissivities from files. + + Notes + ----- + The first time this function is run on a folder it will generate an npy + file containing all the BPASS emissivities for faster loading in the + future. It stores the file in the same folder with the name: + `all_ionizing-[bin/sin]-[imf].npy`. + + The emissivities are just read from file and not normalised. + + + Input + ----- + data_path : `str` + The path to the folder containing the BPASS files. + + binary : `bool` + Use the binary files or just the single stars. Default=True + + imf : `str` + BPASS Identifier of the IMF to be used. + The accepted IMF identifiers are: + - `"imf_chab100"` + - `"imf_chab300"` + - `"imf100_100"` + - `"imf100_300"` + - `"imf135_100"` + - `"imf135_300"` + - `"imfall_300"` + - `"imf170_100"` + - `"imf170_300"` + + Returns + ------- + emissivities : `numpy.ndarray` (N_Z, N_age, 4) [(metallicity, log_age, band)] + A 3D numpy array containing all the BPASS emissivities (Nion [1/s], + L_Halpha [ergs/s], L_FUV [ergs/s/A], L_NUV [ergs/s/A]) for a specific + imf and binary or single star population. + Usage: spectra[1][2][0] + (gives Nion for Z=0.0001 and log_age=6.2) + """ + # Check population type + star = "bin" if binary else "sin" + + # check IMF key + if imf not in BPASS_IMFS: + raise HokiKeyError( + f"{imf} is not a BPASS IMF. Please select a correct IMF.") + + # check if data_path is a string + if not isinstance(data_path, str): + raise HokiTypeError("The folder location is expected to be a string.") + + # Check if compiled spectra are already present in data folder + if os.path.isfile(f"{data_path}/all_ionizing-{star}-{imf}.npy"): + print("Load precompiled file.") + emissivities = np.load(f"{data_path}/all_ionizing-{star}-{imf}.npy") + print("Done Loading.") + + # Compile the spectra for faster reading next time otherwise + else: + print("Compiled file not found. Data will be compiled.") + res = hoki.data_compilers.EmissivityCompiler( + data_path, data_path, imf, binary=binary + ) + emissivities = res.output + return emissivities + + ################# # # ################# diff --git a/hoki/tests/corner_plot.png b/hoki/tests/corner_plot.png new file mode 100644 index 00000000..7a7997ec Binary files /dev/null and b/hoki/tests/corner_plot.png differ diff --git a/hoki/tests/test_data_compiler.py b/hoki/tests/test_data_compilers.py similarity index 73% rename from hoki/tests/test_data_compiler.py rename to hoki/tests/test_data_compilers.py index be229fe8..fc540a3e 100644 --- a/hoki/tests/test_data_compiler.py +++ b/hoki/tests/test_data_compilers.py @@ -4,25 +4,24 @@ Tests for the data_compiler package """ -from hoki.data_compilers import ModelDataCompiler, SpectraCompiler -import hoki.data_compilers as dc +import os.path import pytest -from hoki.utils.exceptions import HokiFatalError, HokiUserWarning, HokiFormatError import numpy as np -import pkg_resources - -import os.path from unittest.mock import patch import numpy.testing as npt +import pkg_resources + +import hoki.data_compilers as dc +from hoki.data_compilers import ModelDataCompiler, SpectraCompiler, EmissivityCompiler from hoki.load import model_output +from hoki.utils.exceptions import HokiFatalError, HokiUserWarning, HokiFormatError data_path = pkg_resources.resource_filename('hoki', 'data') data_path+="/" #models_path=data_path+"sample_stellar_models/" #print(models_path) - class TestSelectInputFiles(object): def test_given_z(self): filename = dc._select_input_files(['z014'])[0] @@ -46,6 +45,7 @@ def test_bad_input_columns(self): with pytest.raises(HokiFormatError): __ = ModelDataCompiler(z_list=['z020'], columns=['bla']) + class TestSpectraCompiler(object): # Initialise model_output DataFrame return a smaller single dataframe @@ -61,7 +61,7 @@ def test_compiler(self, mock_isfile, mock_model_output): # Set the model_output to the DataFrame mock_model_output.return_value = self.data.to_numpy() mock_isfile.return_value = True - + spec = SpectraCompiler(f"{data_path}", f"{data_path}", "imf135_300") @@ -71,7 +71,7 @@ def test_compiler(self, mock_isfile, mock_model_output): # Check output dataframe npt.assert_allclose( - spec.spectra[3], + spec.output[3], self.data.loc[:, slice("6.0", "11.0")].T.to_numpy(), err_msg="Complied spectra is wrong." ) @@ -80,6 +80,37 @@ def test_compiler(self, mock_isfile, mock_model_output): os.remove(f"{data_path}/all_spectra-bin-imf135_300.npy") +class TestEmissivityCompiler(object): + + # Initialise model_output DataFrame return a smaller single dataframe + # This reduces I/O readings + data = model_output( + f"{data_path}/ionizing-bin-imf135_300.z002.dat") + + # Patch the model_output function + @patch("hoki.data_compilers.np.loadtxt") + @patch("hoki.data_compilers.isfile") + def test_compiler(self, mock_isfile, mock_model_output): + + # Set the model_output to the DataFrame + mock_model_output.return_value = self.data.to_numpy() + mock_isfile.return_value = True + + res = EmissivityCompiler(f"{data_path}", + f"{data_path}", + "imf135_300") + + assert os.path.isfile(f"{data_path}/all_ionizing-bin-imf135_300.npy") + + npt.assert_allclose( + res.output[3], + self.data.drop(columns='log_age').to_numpy(), + err_msg="Compiled emissivities is wrong." + ) + os.remove(f"{data_path}/all_ionizing-bin-imf135_300.npy") + + + """ def test_compiling_small_dataset(self): small_set = ModelDataCompiler(z_list=['z020'], diff --git a/hoki/tests/test_load.py b/hoki/tests/test_load.py index b3719b76..80a78ed2 100644 --- a/hoki/tests/test_load.py +++ b/hoki/tests/test_load.py @@ -172,7 +172,7 @@ def test_output(self): "Models are not loaded correctly." -class TestLoadAllsSpectra(object): +class TestLoadAllSpectra(object): # Initialise model_output DataFrame # This reduces I/O readings @@ -186,7 +186,7 @@ def test_compile_spectra(self, mock_isfile, mock_model_output): # Set the model_output to the DataFrame mock_model_output.return_value = self.data.to_numpy() - + mock_isfile.return_value = True spec = load.spectra_all_z(f"{data_path}", "imf135_300") # Check if compiled file is created @@ -212,3 +212,45 @@ def test_load_pickled_file(self): ) os.remove(f"{data_path}/all_spectra-bin-imf135_300.npy") + + +class TestLoadAllEmissivities(object): + + # Initialise model_output DataFrame + # This reduces I/O readings + data = load.model_output( + f"{data_path}/ionizing-bin-imf135_300.z002.dat") + + # Patch the model_output function + @patch("hoki.data_compilers.np.loadtxt") + @patch("hoki.data_compilers.isfile") + def test_compile_emissivities(self, mock_isfile, mock_model_output): + + # Set the model_output to the DataFrame + mock_model_output.return_value = self.data.to_numpy() + mock_isfile.return_value = True + res = load.emissivities_all_z(f"{data_path}", "imf135_300") + + # Check if compiled file is created + assert os.path.isfile(f"{data_path}/all_ionizing-bin-imf135_300.npy"),\ + "No compiled file is created." + + # Check output numpy array + npt.assert_allclose( + res[3], + self.data.drop(columns='log_age').to_numpy(), + err_msg="Loading of files has failed." + ) + + def test_load_pickled_file(self): + + res = load.emissivities_all_z(f"{data_path}", "imf135_300") + + # Check output numpy array + npt.assert_allclose( + res[3], + self.data.drop(columns='log_age').to_numpy(), + err_msg="Loading of compiled file has failed." + ) + + os.remove(f"{data_path}/all_ionizing-bin-imf135_300.npy") diff --git a/setup.cfg b/setup.cfg index e1e63df4..1833c314 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ install_requires = matplotlib pyyaml wheel - pysynphot numba emcee corner