From 8ae752bdc44e32ca08f68b3c91fcf0169f5a625f Mon Sep 17 00:00:00 2001 From: mfegan Date: Tue, 24 Jul 2018 13:11:40 -0500 Subject: [PATCH 1/2] Issue #13299 Asyncronous download of specific parameters to CSV and NetCSF is inconsistent - Insure lat/lon, if available, are included in JSON format - Remove stream name from external parameters in CSV and JSON formats --- util/csvresponse.py | 7 +++++++ util/jsonresponse.py | 10 +++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/util/csvresponse.py b/util/csvresponse.py index ef37a3fa5..acd959b24 100644 --- a/util/csvresponse.py +++ b/util/csvresponse.py @@ -137,6 +137,13 @@ def _create_csv(self, dataset, filehandle): # filter the dataset dataset = dataset.drop(drop.intersection(dataset)) + # rename the parameter without the stream_name prefix (12544 AC1) + for extern in self.stream_request.external_includes: + for param in self.stream_request.external_includes[extern]: + long_name = '-'.join((extern.stream_name, param.name)) + if long_name in dataset.data_vars: + dataset = dataset.rename({long_name : param.name}) + # write the CSV -- note to_csv() returns none dataset.to_dataframe().to_csv(path_or_buf=filehandle, sep=self.delimiter) diff --git a/util/jsonresponse.py b/util/jsonresponse.py index 3b632b259..6d0606082 100644 --- a/util/jsonresponse.py +++ b/util/jsonresponse.py @@ -130,14 +130,18 @@ def _deployment_particles(self, ds, stream_key, parameters, external_includes): if lat_data is not None and lon_data is not None: data['lat'] = lat_data data['lon'] = lon_data - params.extend(('lat', 'lon')) + # always include lat/lon + params.extend(('lat', 'lon')) - # remaining externals + + # remaining externals. rename the parameter without the stream_name prefix (12544 AC1) for sk in external_includes: for param in external_includes[sk]: name = '-'.join((sk.stream_name, param.name)) if name in data: - params.append(name) + extern_data = data.pop(name) + data[param.name] = extern_data + params.append(param.name) if self.stream_request.include_provenance: params.append('provenance') From c23bd2f2a7e8692cd5e4509a7638d7f4e8fdc107 Mon Sep 17 00:00:00 2001 From: mfegan Date: Wed, 25 Jul 2018 10:16:44 -0500 Subject: [PATCH 2/2] Issue #13299 Asyncronous download of specific parameters to CSV and NetCSF is inconsistent - Insure lat/lon, if available, are included in JSON format - remove name change for externals in JSON and CSV packaging --- util/csvresponse.py | 7 ------- util/jsonresponse.py | 6 ++---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/util/csvresponse.py b/util/csvresponse.py index acd959b24..ef37a3fa5 100644 --- a/util/csvresponse.py +++ b/util/csvresponse.py @@ -137,13 +137,6 @@ def _create_csv(self, dataset, filehandle): # filter the dataset dataset = dataset.drop(drop.intersection(dataset)) - # rename the parameter without the stream_name prefix (12544 AC1) - for extern in self.stream_request.external_includes: - for param in self.stream_request.external_includes[extern]: - long_name = '-'.join((extern.stream_name, param.name)) - if long_name in dataset.data_vars: - dataset = dataset.rename({long_name : param.name}) - # write the CSV -- note to_csv() returns none dataset.to_dataframe().to_csv(path_or_buf=filehandle, sep=self.delimiter) diff --git a/util/jsonresponse.py b/util/jsonresponse.py index 6d0606082..6525af96b 100644 --- a/util/jsonresponse.py +++ b/util/jsonresponse.py @@ -134,14 +134,12 @@ def _deployment_particles(self, ds, stream_key, parameters, external_includes): params.extend(('lat', 'lon')) - # remaining externals. rename the parameter without the stream_name prefix (12544 AC1) + # remaining externals. for sk in external_includes: for param in external_includes[sk]: name = '-'.join((sk.stream_name, param.name)) if name in data: - extern_data = data.pop(name) - data[param.name] = extern_data - params.append(param.name) + params.append(name) if self.stream_request.include_provenance: params.append('provenance')